
<!doctype html>
<html lang="en" class="no-js">
  <head>
    
      <meta charset="utf-8">
      <meta name="viewport" content="width=device-width,initial-scale=1">
      
      
      
      <link rel="icon" href="../../../../static/images/favicon.png">
      <meta name="generator" content="mkdocs-1.3.0, mkdocs-material-8.2.8">
    
    
      
        <title>基于 CDH 6 大数据平台生产部署 - WL4G DOCS</title>
      
    
    
      <link rel="stylesheet" href="../../../../assets/stylesheets/main.644de097.min.css">
      
        
        <link rel="stylesheet" href="../../../../assets/stylesheets/palette.e6a45f82.min.css">
        
      
    
    
    
      
        
        
        <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
        <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,300i,400,400i,700,700i%7CRoboto+Mono:400,400i,700,700i&display=fallback">
        <style>:root{--md-text-font:"Roboto";--md-code-font:"Roboto Mono"}</style>
      
    
    
      <link rel="stylesheet" href="../../../../static/css/util.css">
    
    <script>__md_scope=new URL("../../../..",location),__md_get=(e,_=localStorage,t=__md_scope)=>JSON.parse(_.getItem(t.pathname+"."+e)),__md_set=(e,_,t=localStorage,a=__md_scope)=>{try{t.setItem(a.pathname+"."+e,JSON.stringify(_))}catch(e){}}</script>
    
      

    
    
  </head>
  
  
    
    
      
    
    
    
    
    <body dir="ltr" data-md-color-scheme="default" data-md-color-primary="" data-md-color-accent="">
  
    
    
      <script>var palette=__md_get("__palette");if(palette&&"object"==typeof palette.color)for(var key of Object.keys(palette.color))document.body.setAttribute("data-md-color-"+key,palette.color[key])</script>
    
    <input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
    <input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
    <label class="md-overlay" for="__drawer"></label>
    <div data-md-component="skip">
      
        
        <a href="#cdh-6" class="md-skip">
          Skip to content
        </a>
      
    </div>
    <div data-md-component="announce">
      
    </div>
    
      <div data-md-component="outdated" hidden>
        <aside class="md-banner md-banner--warning">
          
        </aside>
      </div>
    
    
      

<header class="md-header" data-md-component="header">
  <nav class="md-header__inner md-grid" aria-label="Header">
    <a href="../../../.." title="WL4G DOCS" class="md-header__button md-logo" aria-label="WL4G DOCS" data-md-component="logo">
      
  <img src="../../../../static/images/mylogo.jpeg" alt="logo">

    </a>
    <label class="md-header__button md-icon" for="__drawer">
      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M3 6h18v2H3V6m0 5h18v2H3v-2m0 5h18v2H3v-2z"/></svg>
    </label>
    <div class="md-header__title" data-md-component="header-title">
      <div class="md-header__ellipsis">
        <div class="md-header__topic">
          <span class="md-ellipsis">
            WL4G DOCS
          </span>
        </div>
        <div class="md-header__topic" data-md-component="header-topic">
          <span class="md-ellipsis">
            
              基于 CDH 6 大数据平台生产部署
            
          </span>
        </div>
      </div>
    </div>
    
      <form class="md-header__option" data-md-component="palette">
        
          
          
          <input class="md-option" data-md-color-media="(prefers-color-scheme: light)" data-md-color-scheme="default" data-md-color-primary="" data-md-color-accent=""  aria-label="Switch to dark mode"  type="radio" name="__palette" id="__palette_1">
          
            <label class="md-header__button md-icon" title="Switch to dark mode" for="__palette_2" hidden>
              <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M17 6H7c-3.31 0-6 2.69-6 6s2.69 6 6 6h10c3.31 0 6-2.69 6-6s-2.69-6-6-6zm0 10H7c-2.21 0-4-1.79-4-4s1.79-4 4-4h10c2.21 0 4 1.79 4 4s-1.79 4-4 4zM7 9c-1.66 0-3 1.34-3 3s1.34 3 3 3 3-1.34 3-3-1.34-3-3-3z"/></svg>
            </label>
          
        
          
          
          <input class="md-option" data-md-color-media="(prefers-color-scheme: dark)" data-md-color-scheme="slate" data-md-color-primary="" data-md-color-accent=""  aria-label="Switch to light mode"  type="radio" name="__palette" id="__palette_2">
          
            <label class="md-header__button md-icon" title="Switch to light mode" for="__palette_1" hidden>
              <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M17 7H7a5 5 0 0 0-5 5 5 5 0 0 0 5 5h10a5 5 0 0 0 5-5 5 5 0 0 0-5-5m0 8a3 3 0 0 1-3-3 3 3 0 0 1 3-3 3 3 0 0 1 3 3 3 3 0 0 1-3 3z"/></svg>
            </label>
          
        
      </form>
    
    
      <div class="md-header__option">
        <div class="md-select">
          
          <button class="md-header__button md-icon" aria-label="Select language">
            <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="m12.87 15.07-2.54-2.51.03-.03A17.52 17.52 0 0 0 14.07 6H17V4h-7V2H8v2H1v2h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04M18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12m-2.62 7 1.62-4.33L19.12 17h-3.24z"/></svg>
          </button>
          <div class="md-select__inner">
            <ul class="md-select__list">
              
                <li class="md-select__item">
                  <a href="/en/" hreflang="en" class="md-select__link">
                    English
                  </a>
                </li>
                
                <li class="md-select__item">
                  <a href="/zh/" hreflang="zh" class="md-select__link">
                    简体中文
                  </a>
                </li>
                
            </ul>
          </div>
        </div>
      </div>
    
    
      <label class="md-header__button md-icon" for="__search">
        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.516 6.516 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5z"/></svg>
      </label>
      <div class="md-search" data-md-component="search" role="dialog">
  <label class="md-search__overlay" for="__search"></label>
  <div class="md-search__inner" role="search">
    <form class="md-search__form" name="search">
      <input type="text" class="md-search__input" name="query" aria-label="Search" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="search-query" required>
      <label class="md-search__icon md-icon" for="__search">
        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.516 6.516 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5z"/></svg>
        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11h12z"/></svg>
      </label>
      <nav class="md-search__options" aria-label="Search">
        
        <button type="reset" class="md-search__icon md-icon" aria-label="Clear" tabindex="-1">
          <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 6.41 17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12 19 6.41z"/></svg>
        </button>
      </nav>
      
        <div class="md-search__suggest" data-md-component="search-suggest"></div>
      
    </form>
    <div class="md-search__output">
      <div class="md-search__scrollwrap" data-md-scrollfix>
        <div class="md-search-result" data-md-component="search-result">
          <div class="md-search-result__meta">
            Initializing search
          </div>
          <ol class="md-search-result__list"></ol>
        </div>
      </div>
    </div>
  </div>
</div>
    
    
  </nav>
  
</header>
    
    <div class="md-container" data-md-component="container">
      
      
        
          
            
<nav class="md-tabs" aria-label="Tabs" data-md-component="tabs">
  <div class="md-tabs__inner md-grid">
    <ul class="md-tabs__list">
      
        
  
  


  
  
  
    <li class="md-tabs__item">
      <a href="../../../.." class="md-tabs__link">
        Getting Started
      </a>
    </li>
  

      
    </ul>
  </div>
</nav>
          
        
      
      <main class="md-main" data-md-component="main">
        <div class="md-main__inner md-grid">
          
            
              
              <div class="md-sidebar md-sidebar--primary" data-md-component="sidebar" data-md-type="navigation" >
                <div class="md-sidebar__scrollwrap">
                  <div class="md-sidebar__inner">
                    

  


  

<nav class="md-nav md-nav--primary md-nav--lifted md-nav--integrated" aria-label="Navigation" data-md-level="0">
  <label class="md-nav__title" for="__drawer">
    <a href="../../../.." title="WL4G DOCS" class="md-nav__button md-logo" aria-label="WL4G DOCS" data-md-component="logo">
      
  <img src="../../../../static/images/mylogo.jpeg" alt="logo">

    </a>
    WL4G DOCS
  </label>
  
  <ul class="md-nav__list" data-md-scrollfix>
    
      
      
      

  
  
  
    
    <li class="md-nav__item md-nav__item--nested">
      
      
        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_1" data-md-state="indeterminate" type="checkbox" id="__nav_1" checked>
      
      
      
      
        <label class="md-nav__link" for="__nav_1">
          Getting Started
          <span class="md-nav__icon md-icon"></span>
        </label>
      
      <nav class="md-nav" aria-label="Getting Started" data-md-level="1">
        <label class="md-nav__title" for="__nav_1">
          <span class="md-nav__icon md-icon"></span>
          Getting Started
        </label>
        <ul class="md-nav__list" data-md-scrollfix>
          
            
              
  
  
  
    <li class="md-nav__item">
      <a href="../../../.." class="md-nav__link">
        Introduction
      </a>
    </li>
  

            
          
            
              
  
  
  
    <li class="md-nav__item">
      <a href="../../../../ABOUT_CN/" class="md-nav__link">
        About
      </a>
    </li>
  

            
          
        </ul>
      </nav>
    </li>
  

    
  </ul>
</nav>
                  </div>
                </div>
              </div>
            
            
          
          <div class="md-content" data-md-component="content">
            <article class="md-content__inner md-typeset">
              
                


<h1 id="cdh-6">基于 CDH 6 大数据平台生产部署<a class="headerlink" href="#cdh-6" title="Permanent link">&para;</a></h1>
<ul>
<li>相关1：<a href="https://blogs.wl4g.com:443/archives/1430">基于 Apache Hadoop 大数据平台生产部署（原汁原味）</a></li>
</ul>
<h2 id="1">1. 部署拓扑<a class="headerlink" href="#1" title="Permanent link">&para;</a></h2>
<table>
<thead>
<tr>
<th>IP</th>
<th>Host</th>
<th>CM Components</th>
<th>CDH Components</th>
</tr>
</thead>
<tbody>
<tr>
<td>10.0.0.111</td>
<td>cdh6-master-1</td>
<td>cloudera-scm-server,  cloudera-scm-agent</td>
<td>QuorumPeerMain//JournalNode/NameNode/</br>HttpFSServerWebServer/ResourceManager/</br>HMaster/ThriftServer/Kafka/HistoryServer/FlinkYarnSessionCli/...</td>
</tr>
<tr>
<td>10.0.0.112</td>
<td>cdh6-worker-1</td>
<td>cloudera-scm-agent</td>
<td>QuorumPeerMain/JournalNode/DataNode/</br>NodeManager/HRegionServer/Kafka/FlinkYarnSessionCli/...</td>
</tr>
<tr>
<td>10.0.0.113</td>
<td>cdh6-worker-2</td>
<td>cloudera-scm-agent</td>
<td>QuorumPeerMain/JournalNode/DataNode/</br>NodeManager/HRegionServer/Kafka/FlinkYarnSessionCli/...</td>
</tr>
</tbody>
</table>
<h2 id="2-os">2. OS 调优<a class="headerlink" href="#2-os" title="Permanent link">&para;</a></h2>
<ul>
<li>kernel 调优</li>
</ul>
<div class="highlight"><pre><span></span><code><a id="__codelineno-0-1" name="__codelineno-0-1"></a><a href="#__codelineno-0-1"><span class="linenos" data-linenos=" 1 "></span></a><span class="c1"># 关闭 selinux</span>
<a id="__codelineno-0-2" name="__codelineno-0-2"></a><a href="#__codelineno-0-2"><span class="linenos" data-linenos=" 2 "></span></a>sudo getenforce
<a id="__codelineno-0-3" name="__codelineno-0-3"></a><a href="#__codelineno-0-3"><span class="linenos" data-linenos=" 3 "></span></a>sudo setenforce <span class="m">0</span>
<a id="__codelineno-0-4" name="__codelineno-0-4"></a><a href="#__codelineno-0-4"><span class="linenos" data-linenos=" 4 "></span></a>
<a id="__codelineno-0-5" name="__codelineno-0-5"></a><a href="#__codelineno-0-5"><span class="linenos" data-linenos=" 5 "></span></a><span class="c1"># 禁止swap</span>
<a id="__codelineno-0-6" name="__codelineno-0-6"></a><a href="#__codelineno-0-6"><span class="linenos" data-linenos=" 6 "></span></a>sudo sysctl -w vm.swappiness<span class="o">=</span><span class="m">0</span>
<a id="__codelineno-0-7" name="__codelineno-0-7"></a><a href="#__codelineno-0-7"><span class="linenos" data-linenos=" 7 "></span></a>sudo <span class="nb">echo</span> <span class="s1">&#39;vm.swappiness=0&#39;</span> &gt;&gt; /etc/sysctl.conf
<a id="__codelineno-0-8" name="__codelineno-0-8"></a><a href="#__codelineno-0-8"><span class="linenos" data-linenos=" 8 "></span></a>sudo sysctl -p
<a id="__codelineno-0-9" name="__codelineno-0-9"></a><a href="#__codelineno-0-9"><span class="linenos" data-linenos=" 9 "></span></a>
<a id="__codelineno-0-10" name="__codelineno-0-10"></a><a href="#__codelineno-0-10"><span class="linenos" data-linenos="10 "></span></a><span class="c1"># 禁止透明大页</span>
<a id="__codelineno-0-11" name="__codelineno-0-11"></a><a href="#__codelineno-0-11"><span class="linenos" data-linenos="11 "></span></a>sudo <span class="nb">echo</span> never &gt; /sys/kernel/mm/transparent_hugepage/enabled
<a id="__codelineno-0-12" name="__codelineno-0-12"></a><a href="#__codelineno-0-12"><span class="linenos" data-linenos="12 "></span></a>sudo <span class="nb">echo</span> never &gt; /sys/kernel/mm/transparent_hugepage/defrag
<a id="__codelineno-0-13" name="__codelineno-0-13"></a><a href="#__codelineno-0-13"><span class="linenos" data-linenos="13 "></span></a>
<a id="__codelineno-0-14" name="__codelineno-0-14"></a><a href="#__codelineno-0-14"><span class="linenos" data-linenos="14 "></span></a><span class="c1"># see:https://blogs.wl4g.com/archives/1267</span>
<a id="__codelineno-0-15" name="__codelineno-0-15"></a><a href="#__codelineno-0-15"><span class="linenos" data-linenos="15 "></span></a>sudo yum install -y ntp
<a id="__codelineno-0-16" name="__codelineno-0-16"></a><a href="#__codelineno-0-16"><span class="linenos" data-linenos="16 "></span></a>sudo timedatectl set-timezone Asia/Shanghai
</code></pre></div>
<ul>
<li>禁用 ipv6</li>
</ul>
<div class="highlight"><pre><span></span><code><a id="__codelineno-1-1" name="__codelineno-1-1"></a><a href="#__codelineno-1-1"><span class="linenos" data-linenos=" 1 "></span></a><span class="c1"># -----临时禁用所有接口 ipv6. -----</span>
<a id="__codelineno-1-2" name="__codelineno-1-2"></a><a href="#__codelineno-1-2"><span class="linenos" data-linenos=" 2 "></span></a><span class="k">for</span> f <span class="k">in</span> <span class="k">$(</span>ls /proc/sys/net/ipv6/conf/*/disable_ipv6<span class="k">)</span><span class="p">;</span> <span class="k">do</span> sudo su - root -c <span class="s2">&quot;echo 1 &gt;</span><span class="nv">$f</span><span class="s2">&quot;</span><span class="p">;</span> <span class="k">done</span>
<a id="__codelineno-1-3" name="__codelineno-1-3"></a><a href="#__codelineno-1-3"><span class="linenos" data-linenos=" 3 "></span></a><span class="c1"># 再次查看所有接口 ipv6 启用状态</span>
<a id="__codelineno-1-4" name="__codelineno-1-4"></a><a href="#__codelineno-1-4"><span class="linenos" data-linenos=" 4 "></span></a>cat /proc/sys/net/ipv6/conf/*/disable_ipv6
<a id="__codelineno-1-5" name="__codelineno-1-5"></a><a href="#__codelineno-1-5"><span class="linenos" data-linenos=" 5 "></span></a>
<a id="__codelineno-1-6" name="__codelineno-1-6"></a><a href="#__codelineno-1-6"><span class="linenos" data-linenos=" 6 "></span></a><span class="c1"># ----- 永久禁用所有接口 ipv6. -----</span>
<a id="__codelineno-1-7" name="__codelineno-1-7"></a><a href="#__codelineno-1-7"><span class="linenos" data-linenos=" 7 "></span></a>sudo cat <span class="s">&lt;&lt;-&#39;EOF&#39; &gt;/etc/sysctl.d/99-ipv6-disable.conf</span>
<a id="__codelineno-1-8" name="__codelineno-1-8"></a><a href="#__codelineno-1-8"><span class="linenos" data-linenos=" 8 "></span></a><span class="s">net.ipv6.conf.all.disable_ipv6=1</span>
<a id="__codelineno-1-9" name="__codelineno-1-9"></a><a href="#__codelineno-1-9"><span class="linenos" data-linenos=" 9 "></span></a><span class="s">net.ipv6.conf.default.disable_ipv6=1</span>
<a id="__codelineno-1-10" name="__codelineno-1-10"></a><a href="#__codelineno-1-10"><span class="linenos" data-linenos="10 "></span></a><span class="s">EOF</span>
<a id="__codelineno-1-11" name="__codelineno-1-11"></a><a href="#__codelineno-1-11"><span class="linenos" data-linenos="11 "></span></a>
<a id="__codelineno-1-12" name="__codelineno-1-12"></a><a href="#__codelineno-1-12"><span class="linenos" data-linenos="12 "></span></a><span class="c1"># 禁用每个接口 ipv6 (虽然设置了 net.ipv6.conf.all.disable_ipv6=1</span>
<a id="__codelineno-1-13" name="__codelineno-1-13"></a><a href="#__codelineno-1-13"><span class="linenos" data-linenos="13 "></span></a><span class="c1"># 但默认对每个接口, 如: net.ipv6.conf.enp3s0.disable_ipv6=0) 还是启用的.</span>
<a id="__codelineno-1-14" name="__codelineno-1-14"></a><a href="#__codelineno-1-14"><span class="linenos" data-linenos="14 "></span></a>
<a id="__codelineno-1-15" name="__codelineno-1-15"></a><a href="#__codelineno-1-15"><span class="linenos" data-linenos="15 "></span></a><span class="k">for</span> i <span class="k">in</span> <span class="k">$(</span>ifconfig <span class="p">|</span> awk -F<span class="s1">&#39;[ :]+&#39;</span> <span class="s1">&#39;!NF{if(eth!=&quot;&quot;&amp;&amp;ip==&quot;&quot;)print eth;eth=ip4=&quot;&quot;}/^[^ ]/{eth=$1}/inet addr:/{ip=$4}&#39;</span><span class="k">)</span><span class="p">;</span> <span class="k">do</span> <span class="nb">echo</span> <span class="s2">&quot;net.ipv6.conf.</span><span class="nv">$i</span><span class="s2">.disable_ipv6=1&quot;</span> &gt;&gt;/etc/sysctl.d/99-ipv6-disable.conf<span class="p">;</span> <span class="k">done</span>
<a id="__codelineno-1-16" name="__codelineno-1-16"></a><a href="#__codelineno-1-16"><span class="linenos" data-linenos="16 "></span></a>
<a id="__codelineno-1-17" name="__codelineno-1-17"></a><a href="#__codelineno-1-17"><span class="linenos" data-linenos="17 "></span></a>sudo sysctl -p
</code></pre></div>
<ul>
<li>关闭 firewalld</li>
</ul>
<div class="highlight"><pre><span></span><code><a id="__codelineno-2-1" name="__codelineno-2-1"></a><a href="#__codelineno-2-1"><span class="linenos" data-linenos="1 "></span></a>systemctl stop firewalld
<a id="__codelineno-2-2" name="__codelineno-2-2"></a><a href="#__codelineno-2-2"><span class="linenos" data-linenos="2 "></span></a>systemctl disable firewalld
</code></pre></div>
<h2 id="3-mysql">3. 部署 MySQL<a class="headerlink" href="#3-mysql" title="Permanent link">&para;</a></h2>
<ul>
<li>
<p>配置 my.cnf</p>
</li>
<li>
<p>参见: <a href="https://blogs.wl4g.com/archives/650">https://blogs.wl4g.com/archives/650</a></p>
</li>
</ul>
<blockquote>
<p>注：不能开启 gtid，因为 cmf 存在无主键表</p>
</blockquote>
<div class="highlight"><pre><span></span><code><a id="__codelineno-3-1" name="__codelineno-3-1"></a><a href="#__codelineno-3-1"><span class="linenos" data-linenos=" 1 "></span></a>sudo cat <span class="s">&lt;&lt;-EOF &gt; /etc/my.cnf</span>
<a id="__codelineno-3-2" name="__codelineno-3-2"></a><a href="#__codelineno-3-2"><span class="linenos" data-linenos=" 2 "></span></a><span class="s"># see:https://blogs.wl4g.com/archives/650</span>
<a id="__codelineno-3-3" name="__codelineno-3-3"></a><a href="#__codelineno-3-3"><span class="linenos" data-linenos=" 3 "></span></a><span class="s">[mysqld]</span>
<a id="__codelineno-3-4" name="__codelineno-3-4"></a><a href="#__codelineno-3-4"><span class="linenos" data-linenos=" 4 "></span></a><span class="s">server_id = 1</span>
<a id="__codelineno-3-5" name="__codelineno-3-5"></a><a href="#__codelineno-3-5"><span class="linenos" data-linenos=" 5 "></span></a><span class="s">port = 3306</span>
<a id="__codelineno-3-6" name="__codelineno-3-6"></a><a href="#__codelineno-3-6"><span class="linenos" data-linenos=" 6 "></span></a>
<a id="__codelineno-3-7" name="__codelineno-3-7"></a><a href="#__codelineno-3-7"><span class="linenos" data-linenos=" 7 "></span></a><span class="s">lower_case_table_names = 1</span>
<a id="__codelineno-3-8" name="__codelineno-3-8"></a><a href="#__codelineno-3-8"><span class="linenos" data-linenos=" 8 "></span></a><span class="s">sql_mode = NO_ZERO_IN_DATE,NO_ZERO_DATE,NO_AUTO_CREATE_USER,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION,STRICT_TRANS_TABLES</span>
<a id="__codelineno-3-9" name="__codelineno-3-9"></a><a href="#__codelineno-3-9"><span class="linenos" data-linenos=" 9 "></span></a><span class="s">explicit_defaults_for_timestamp = true</span>
<a id="__codelineno-3-10" name="__codelineno-3-10"></a><a href="#__codelineno-3-10"><span class="linenos" data-linenos="10 "></span></a>
<a id="__codelineno-3-11" name="__codelineno-3-11"></a><a href="#__codelineno-3-11"><span class="linenos" data-linenos="11 "></span></a><span class="s">symbolic-links = 0</span>
<a id="__codelineno-3-12" name="__codelineno-3-12"></a><a href="#__codelineno-3-12"><span class="linenos" data-linenos="12 "></span></a><span class="s">## Note: </span>
<a id="__codelineno-3-13" name="__codelineno-3-13"></a><a href="#__codelineno-3-13"><span class="linenos" data-linenos="13 "></span></a><span class="s">#innodb_buffer_pool_size = 1G</span>
<a id="__codelineno-3-14" name="__codelineno-3-14"></a><a href="#__codelineno-3-14"><span class="linenos" data-linenos="14 "></span></a><span class="s">#innodb_buffer_pool_instances = 8</span>
<a id="__codelineno-3-15" name="__codelineno-3-15"></a><a href="#__codelineno-3-15"><span class="linenos" data-linenos="15 "></span></a><span class="s">#innodb_buffer_pool_chunk_size = 128M</span>
<a id="__codelineno-3-16" name="__codelineno-3-16"></a><a href="#__codelineno-3-16"><span class="linenos" data-linenos="16 "></span></a><span class="s">max_allowed_packet = 1G</span>
<a id="__codelineno-3-17" name="__codelineno-3-17"></a><a href="#__codelineno-3-17"><span class="linenos" data-linenos="17 "></span></a><span class="s">max_connections = 2000</span>
<a id="__codelineno-3-18" name="__codelineno-3-18"></a><a href="#__codelineno-3-18"><span class="linenos" data-linenos="18 "></span></a><span class="s">slave_max_allowed_packet = 1G</span>
<a id="__codelineno-3-19" name="__codelineno-3-19"></a><a href="#__codelineno-3-19"><span class="linenos" data-linenos="19 "></span></a>
<a id="__codelineno-3-20" name="__codelineno-3-20"></a><a href="#__codelineno-3-20"><span class="linenos" data-linenos="20 "></span></a><span class="s">binlog_format = ROW</span>
<a id="__codelineno-3-21" name="__codelineno-3-21"></a><a href="#__codelineno-3-21"><span class="linenos" data-linenos="21 "></span></a><span class="s">binlog_checksum = NONE</span>
<a id="__codelineno-3-22" name="__codelineno-3-22"></a><a href="#__codelineno-3-22"><span class="linenos" data-linenos="22 "></span></a><span class="s">log_bin = binlog</span>
<a id="__codelineno-3-23" name="__codelineno-3-23"></a><a href="#__codelineno-3-23"><span class="linenos" data-linenos="23 "></span></a><span class="s">log_slave_updates = ON</span>
<a id="__codelineno-3-24" name="__codelineno-3-24"></a><a href="#__codelineno-3-24"><span class="linenos" data-linenos="24 "></span></a><span class="s">relay_log_info_repository = TABLE</span>
<a id="__codelineno-3-25" name="__codelineno-3-25"></a><a href="#__codelineno-3-25"><span class="linenos" data-linenos="25 "></span></a><span class="s">max_binlog_size = 1G</span>
<a id="__codelineno-3-26" name="__codelineno-3-26"></a><a href="#__codelineno-3-26"><span class="linenos" data-linenos="26 "></span></a><span class="s">EOF</span>
</code></pre></div>
<ul>
<li>初始化 mysql</li>
</ul>
<div class="highlight"><pre><span></span><code><a id="__codelineno-4-1" name="__codelineno-4-1"></a><a href="#__codelineno-4-1"><span class="linenos" data-linenos="1 "></span></a><span class="nv">$MYSQL_HOME</span>/bin/mysqladmin -u root -p password --socket<span class="o">=</span>/mnt/disk1/mysql/mysqld.sock
<a id="__codelineno-4-2" name="__codelineno-4-2"></a><a href="#__codelineno-4-2"><span class="linenos" data-linenos="2 "></span></a><span class="nv">$MYSQL_HOME</span>/bin/mysql -S <span class="nv">$MYSQL_SOCKET</span> -uroot -proot
<a id="__codelineno-4-3" name="__codelineno-4-3"></a><a href="#__codelineno-4-3"><span class="linenos" data-linenos="3 "></span></a>
<a id="__codelineno-4-4" name="__codelineno-4-4"></a><a href="#__codelineno-4-4"><span class="linenos" data-linenos="4 "></span></a>create database cmf character <span class="nb">set</span> utf8 collate utf8_bin<span class="p">;</span>
</code></pre></div>
<h2 id="4-yum">4. 部署 yum 仓库<a class="headerlink" href="#4-yum" title="Permanent link">&para;</a></h2>
<ul>
<li>4.1 创建 yum parcel 库目录</li>
</ul>
<div class="highlight"><pre><span></span><code><a id="__codelineno-5-1" name="__codelineno-5-1"></a><a href="#__codelineno-5-1"><span class="linenos" data-linenos=" 1 "></span></a>mkdir -p /usr/share/nginx/html/el7-repo/
<a id="__codelineno-5-2" name="__codelineno-5-2"></a><a href="#__codelineno-5-2"><span class="linenos" data-linenos=" 2 "></span></a>tree /usr/share/nginx/html/el7-repo/
<a id="__codelineno-5-3" name="__codelineno-5-3"></a><a href="#__codelineno-5-3"><span class="linenos" data-linenos=" 3 "></span></a>/usr/share/nginx/html/el7-repo/
<a id="__codelineno-5-4" name="__codelineno-5-4"></a><a href="#__codelineno-5-4"><span class="linenos" data-linenos=" 4 "></span></a>├── cdh6
<a id="__codelineno-5-5" name="__codelineno-5-5"></a><a href="#__codelineno-5-5"><span class="linenos" data-linenos=" 5 "></span></a>│             ├── CDH-6.3.1-1.cdh6.3.1.p0.1470567-el7.parcel
<a id="__codelineno-5-6" name="__codelineno-5-6"></a><a href="#__codelineno-5-6"><span class="linenos" data-linenos=" 6 "></span></a>│             ├── CDH-6.3.1-1.cdh6.3.1.p0.1470567-el7.parcel.sha1
<a id="__codelineno-5-7" name="__codelineno-5-7"></a><a href="#__codelineno-5-7"><span class="linenos" data-linenos=" 7 "></span></a>│             └── manifest.json
<a id="__codelineno-5-8" name="__codelineno-5-8"></a><a href="#__codelineno-5-8"><span class="linenos" data-linenos=" 8 "></span></a>└── cm6
<a id="__codelineno-5-9" name="__codelineno-5-9"></a><a href="#__codelineno-5-9"><span class="linenos" data-linenos=" 9 "></span></a>    └── <span class="m">6</span>.3.1
<a id="__codelineno-5-10" name="__codelineno-5-10"></a><a href="#__codelineno-5-10"><span class="linenos" data-linenos="10 "></span></a>        ├── allkeys.asc
<a id="__codelineno-5-11" name="__codelineno-5-11"></a><a href="#__codelineno-5-11"><span class="linenos" data-linenos="11 "></span></a>        ├── repodata
<a id="__codelineno-5-12" name="__codelineno-5-12"></a><a href="#__codelineno-5-12"><span class="linenos" data-linenos="12 "></span></a>        │   ├── 3662f97de72fd44c017bb0e25cee3bc9398108c8efb745def12130a69df2ecb2-filelists.sqlite.bz2
<a id="__codelineno-5-13" name="__codelineno-5-13"></a><a href="#__codelineno-5-13"><span class="linenos" data-linenos="13 "></span></a>        │   ├── 43f3725f730ee7522712039982aa4befadae4db968c8d780c8eb15ae9872cd4d-primary.xml.gz
<a id="__codelineno-5-14" name="__codelineno-5-14"></a><a href="#__codelineno-5-14"><span class="linenos" data-linenos="14 "></span></a>        │   ├── 49e4d60647407a36819f1d8ed901258a13361749b742e3be9065025ad31feb8e-filelists.xml.gz
<a id="__codelineno-5-15" name="__codelineno-5-15"></a><a href="#__codelineno-5-15"><span class="linenos" data-linenos="15 "></span></a>        │   ├── 8afda99b921fd1538dd06355952719652654fc06b6cd14515437bda28376c03d-other.sqlite.bz2
<a id="__codelineno-5-16" name="__codelineno-5-16"></a><a href="#__codelineno-5-16"><span class="linenos" data-linenos="16 "></span></a>        │   ├── b9300879675bdbc300436c1131a910a535b8b5a5dc6f38e956d51769b6771a96-primary.sqlite.bz2
<a id="__codelineno-5-17" name="__codelineno-5-17"></a><a href="#__codelineno-5-17"><span class="linenos" data-linenos="17 "></span></a>        │   ├── e28836e19e07f71480c4dad0f7a87a804dc93970ec5277ad95614e8ffcff0d58-other.xml.gz
<a id="__codelineno-5-18" name="__codelineno-5-18"></a><a href="#__codelineno-5-18"><span class="linenos" data-linenos="18 "></span></a>        │   ├── repomd.xml
<a id="__codelineno-5-19" name="__codelineno-5-19"></a><a href="#__codelineno-5-19"><span class="linenos" data-linenos="19 "></span></a>        │   ├── repomd.xml.asc
<a id="__codelineno-5-20" name="__codelineno-5-20"></a><a href="#__codelineno-5-20"><span class="linenos" data-linenos="20 "></span></a>        │   └── repomd.xml.key
<a id="__codelineno-5-21" name="__codelineno-5-21"></a><a href="#__codelineno-5-21"><span class="linenos" data-linenos="21 "></span></a>        ├── RPM-GPG-KEY-cloudera
<a id="__codelineno-5-22" name="__codelineno-5-22"></a><a href="#__codelineno-5-22"><span class="linenos" data-linenos="22 "></span></a>        ├── RPMS
<a id="__codelineno-5-23" name="__codelineno-5-23"></a><a href="#__codelineno-5-23"><span class="linenos" data-linenos="23 "></span></a>        │   ├── noarch
<a id="__codelineno-5-24" name="__codelineno-5-24"></a><a href="#__codelineno-5-24"><span class="linenos" data-linenos="24 "></span></a>        │   └── x86_64
<a id="__codelineno-5-25" name="__codelineno-5-25"></a><a href="#__codelineno-5-25"><span class="linenos" data-linenos="25 "></span></a>        │       ├── cloudera-manager-agent-6.3.1-1466458.el7.x86_64.rpm
<a id="__codelineno-5-26" name="__codelineno-5-26"></a><a href="#__codelineno-5-26"><span class="linenos" data-linenos="26 "></span></a>        │       ├── cloudera-manager-daemons-6.3.1-1466458.el7.x86_64.rpm
<a id="__codelineno-5-27" name="__codelineno-5-27"></a><a href="#__codelineno-5-27"><span class="linenos" data-linenos="27 "></span></a>        │       ├── cloudera-manager-server-6.3.1-1466458.el7.x86_64.rpm
<a id="__codelineno-5-28" name="__codelineno-5-28"></a><a href="#__codelineno-5-28"><span class="linenos" data-linenos="28 "></span></a>        │       ├── cloudera-manager-server-db-2-6.3.1-1466458.el7.x86_64.rpm
<a id="__codelineno-5-29" name="__codelineno-5-29"></a><a href="#__codelineno-5-29"><span class="linenos" data-linenos="29 "></span></a>        │       ├── enterprise-debuginfo-6.3.1-1466458.el7.x86_64.rpm
<a id="__codelineno-5-30" name="__codelineno-5-30"></a><a href="#__codelineno-5-30"><span class="linenos" data-linenos="30 "></span></a>        │       └── oracle-j2sdk1.8-1.8.0+update181-1.x86_64.rpm
<a id="__codelineno-5-31" name="__codelineno-5-31"></a><a href="#__codelineno-5-31"><span class="linenos" data-linenos="31 "></span></a>        └── SRPMS
<a id="__codelineno-5-32" name="__codelineno-5-32"></a><a href="#__codelineno-5-32"><span class="linenos" data-linenos="32 "></span></a>└── parcel-flink <span class="c1"># 当集成了自定义构建的 FLINK 才需要</span>
<a id="__codelineno-5-33" name="__codelineno-5-33"></a><a href="#__codelineno-5-33"><span class="linenos" data-linenos="33 "></span></a>    ├── <span class="m">1</span>.11.2
<a id="__codelineno-5-34" name="__codelineno-5-34"></a><a href="#__codelineno-5-34"><span class="linenos" data-linenos="34 "></span></a>    │   ├── flink-1.11.2-bin-scala_2.12-el7.parcel
<a id="__codelineno-5-35" name="__codelineno-5-35"></a><a href="#__codelineno-5-35"><span class="linenos" data-linenos="35 "></span></a>    │   ├── flink-1.11.2-bin-scala_2.12-el7.parcel.sha
<a id="__codelineno-5-36" name="__codelineno-5-36"></a><a href="#__codelineno-5-36"><span class="linenos" data-linenos="36 "></span></a>    │   └── manifest.json
<a id="__codelineno-5-37" name="__codelineno-5-37"></a><a href="#__codelineno-5-37"><span class="linenos" data-linenos="37 "></span></a>    └── <span class="m">1</span>.14.4
<a id="__codelineno-5-38" name="__codelineno-5-38"></a><a href="#__codelineno-5-38"><span class="linenos" data-linenos="38 "></span></a>        ├── flink-1.14.4-bin-scala_2.11-el7.parcel
<a id="__codelineno-5-39" name="__codelineno-5-39"></a><a href="#__codelineno-5-39"><span class="linenos" data-linenos="39 "></span></a>        ├── flink-1.14.4-bin-scala_2.11-el7.parcel.sha
<a id="__codelineno-5-40" name="__codelineno-5-40"></a><a href="#__codelineno-5-40"><span class="linenos" data-linenos="40 "></span></a>        └── manifest.json
<a id="__codelineno-5-41" name="__codelineno-5-41"></a><a href="#__codelineno-5-41"><span class="linenos" data-linenos="41 "></span></a>
<a id="__codelineno-5-42" name="__codelineno-5-42"></a><a href="#__codelineno-5-42"><span class="linenos" data-linenos="42 "></span></a><span class="m">12</span> directories, <span class="m">26</span> files
</code></pre></div>
<ul>
<li>4.2 添加 yum 源 &amp; 安装</li>
</ul>
<div class="highlight"><pre><span></span><code><a id="__codelineno-6-1" name="__codelineno-6-1"></a><a href="#__codelineno-6-1"><span class="linenos" data-linenos=" 1 "></span></a>cat <span class="s">&lt;&lt;-EOF &gt;/etc/yum.repos.d/cloudera.repo</span>
<a id="__codelineno-6-2" name="__codelineno-6-2"></a><a href="#__codelineno-6-2"><span class="linenos" data-linenos=" 2 "></span></a><span class="s">[cloudera-repo]</span>
<a id="__codelineno-6-3" name="__codelineno-6-3"></a><a href="#__codelineno-6-3"><span class="linenos" data-linenos=" 3 "></span></a><span class="s">name=cloudera-repo</span>
<a id="__codelineno-6-4" name="__codelineno-6-4"></a><a href="#__codelineno-6-4"><span class="linenos" data-linenos=" 4 "></span></a><span class="s">baseurl=http://el7-repo.wl4g.io/cm6/6.3.1/</span>
<a id="__codelineno-6-5" name="__codelineno-6-5"></a><a href="#__codelineno-6-5"><span class="linenos" data-linenos=" 5 "></span></a><span class="s">enabled=1</span>
<a id="__codelineno-6-6" name="__codelineno-6-6"></a><a href="#__codelineno-6-6"><span class="linenos" data-linenos=" 6 "></span></a><span class="s">gpgcheck=0</span>
<a id="__codelineno-6-7" name="__codelineno-6-7"></a><a href="#__codelineno-6-7"><span class="linenos" data-linenos=" 7 "></span></a><span class="s">EOF</span>
<a id="__codelineno-6-8" name="__codelineno-6-8"></a><a href="#__codelineno-6-8"><span class="linenos" data-linenos=" 8 "></span></a>
<a id="__codelineno-6-9" name="__codelineno-6-9"></a><a href="#__codelineno-6-9"><span class="linenos" data-linenos=" 9 "></span></a><span class="c1"># 所有节点</span>
<a id="__codelineno-6-10" name="__codelineno-6-10"></a><a href="#__codelineno-6-10"><span class="linenos" data-linenos="10 "></span></a>sudo yum install -y java-1.8.0-openjdk-devel.x86_64 net-tools
<a id="__codelineno-6-11" name="__codelineno-6-11"></a><a href="#__codelineno-6-11"><span class="linenos" data-linenos="11 "></span></a>sudo yum -y install cloudera-manager-daemons cloudera-manager-agent
<a id="__codelineno-6-12" name="__codelineno-6-12"></a><a href="#__codelineno-6-12"><span class="linenos" data-linenos="12 "></span></a><span class="c1"># 仅 master 节点</span>
<a id="__codelineno-6-13" name="__codelineno-6-13"></a><a href="#__codelineno-6-13"><span class="linenos" data-linenos="13 "></span></a>sudo yum -y install cloudera-manager-daemons cloudera-manager-agent cloudera-manager-server
</code></pre></div>
<ul>
<li>4.3 初始化 cmf 库</li>
</ul>
<div class="highlight"><pre><span></span><code><a id="__codelineno-7-1" name="__codelineno-7-1"></a><a href="#__codelineno-7-1"><span class="linenos" data-linenos="1 "></span></a>/opt/cloudera/cm/schema/scm_prepare_database.sh mysql cmf root root
</code></pre></div>
<blockquote>
<p>注: 初始化时若提示<code>All done, your SCM database is configured correctly!</code> 但实际上没有并未初始化，恭喜这踩到了 cdh 6.3.1(之后哪个高版本已解决，就暂未测试了，因为已收费:)) 的 bug，这通常发生在重复初始化时出现，第一次是OK的，如果出现此问题时，可使用这份初始化成功的脚本：<a href="https://gitee.com/wl4g/blogs/raw/master/docs/articles/hadoop/cdh6.3.1-production-deployment/resources/cdh6.3.1_cmf_init.sql">cdh6.3.1_cmf_init.sql</a></p>
<p><strong><em>注：</em></strong> 不要开启 MySQL 的 gtid 复制时 (如 MGR 集群)，会导致cm-server运行报错无法插入数据<code>The table does not comply with the requirements by an external plugin.</code>，原因是这几张表无主键(role_staleness_status/commands_detail/cm_version/client_configs_to_hosts/schema_version)。
尝试过给这几张表第一个字段设置为主键、或添加一个ID字段自增长都会报错。暂无任何解决方案！（不开源改不了代码/(ㄒoㄒ)/~~）</p>
</blockquote>
<ul>
<li>配置 mysql jdbc 驱动</li>
</ul>
<div class="highlight"><pre><span></span><code><a id="__codelineno-8-1" name="__codelineno-8-1"></a><a href="#__codelineno-8-1"><span class="linenos" data-linenos="1 "></span></a>sudo ln -snf /opt/apps/mysql-connector-java-5.1.47.jar /usr/share/java/mysql-connector-java.jar
</code></pre></div>
<ul>
<li>4.4 配置 CM 连接 mysql</li>
</ul>
<blockquote>
<p>解决问题：<code>WARN: Establishing SSL connection without server's identity verification is not recommended. According to MySQL 5.5.45+, 5.6.26+ and 5.7.6+ requirements SSL connection must be established by default if explicit option isn't set. For compliance with existing applications not using SSL the verifyServerCertificate property is set to 'false'. You need either to explicitly disable SSL by setting useSSL=false, or set useSSL=true and provide truststore for server certificate verification.</code></p>
</blockquote>
<div class="highlight"><pre><span></span><code><a id="__codelineno-9-1" name="__codelineno-9-1"></a><a href="#__codelineno-9-1"><span class="linenos" data-linenos=" 1 "></span></a>cat &lt;&lt;EOF &gt;&gt;/etc/cloudera-scm-server/db.properties
<a id="__codelineno-9-2" name="__codelineno-9-2"></a><a href="#__codelineno-9-2"><span class="linenos" data-linenos=" 2 "></span></a># For information describing how to configure the Cloudera Manager Server
<a id="__codelineno-9-3" name="__codelineno-9-3"></a><a href="#__codelineno-9-3"><span class="linenos" data-linenos=" 3 "></span></a># to connect to databases, see the &quot;Cloudera Manager Installation Guide.&quot;
<a id="__codelineno-9-4" name="__codelineno-9-4"></a><a href="#__codelineno-9-4"><span class="linenos" data-linenos=" 4 "></span></a>#
<a id="__codelineno-9-5" name="__codelineno-9-5"></a><a href="#__codelineno-9-5"><span class="linenos" data-linenos=" 5 "></span></a>com.cloudera.cmf.db.type=mysql
<a id="__codelineno-9-6" name="__codelineno-9-6"></a><a href="#__codelineno-9-6"><span class="linenos" data-linenos=" 6 "></span></a>com.cloudera.cmf.db.host=localhost
<a id="__codelineno-9-7" name="__codelineno-9-7"></a><a href="#__codelineno-9-7"><span class="linenos" data-linenos=" 7 "></span></a>com.cloudera.cmf.db.name=cmf
<a id="__codelineno-9-8" name="__codelineno-9-8"></a><a href="#__codelineno-9-8"><span class="linenos" data-linenos=" 8 "></span></a>com.cloudera.cmf.db.user=root
<a id="__codelineno-9-9" name="__codelineno-9-9"></a><a href="#__codelineno-9-9"><span class="linenos" data-linenos=" 9 "></span></a>com.cloudera.cmf.db.setupType=EXTERNAL
<a id="__codelineno-9-10" name="__codelineno-9-10"></a><a href="#__codelineno-9-10"><span class="linenos" data-linenos="10 "></span></a>com.cloudera.cmf.db.password=root
<a id="__codelineno-9-11" name="__codelineno-9-11"></a><a href="#__codelineno-9-11"><span class="linenos" data-linenos="11 "></span></a># see:https://community.cloudera.com/t5/Support-Questions/Cloudera-6-3-MySql-5-7-SSL-Warning/td-p/316592
<a id="__codelineno-9-12" name="__codelineno-9-12"></a><a href="#__codelineno-9-12"><span class="linenos" data-linenos="12 "></span></a>com.cloudera.cmf.orm.hibernate.connection.driver_class=com.mysql.jdbc.Driver
<a id="__codelineno-9-13" name="__codelineno-9-13"></a><a href="#__codelineno-9-13"><span class="linenos" data-linenos="13 "></span></a>com.cloudera.cmf.orm.hibernate.connection.url=jdbc:mysql://127.0.0.1:3306/cmf?useUnicode=true&amp;characterEncoding=UTF-8&amp;useSSL=false
<a id="__codelineno-9-14" name="__codelineno-9-14"></a><a href="#__codelineno-9-14"><span class="linenos" data-linenos="14 "></span></a>#com.cloudera.cmf.db.useSSL=false
<a id="__codelineno-9-15" name="__codelineno-9-15"></a><a href="#__codelineno-9-15"><span class="linenos" data-linenos="15 "></span></a>#com.cloudera.cmf.db.verifyServerCertificate=false
<a id="__codelineno-9-16" name="__codelineno-9-16"></a><a href="#__codelineno-9-16"><span class="linenos" data-linenos="16 "></span></a>#com.cloudera.cmf.db.trustCertificateKeyStoreUrl=file:/usr/java/jdk1.8.0 _121-cloudera/jre/lib/security/jssecacerts
<a id="__codelineno-9-17" name="__codelineno-9-17"></a><a href="#__codelineno-9-17"><span class="linenos" data-linenos="17 "></span></a>#com.cloudera.cmf.db.trustCertificateKeyStoreType=JKS
<a id="__codelineno-9-18" name="__codelineno-9-18"></a><a href="#__codelineno-9-18"><span class="linenos" data-linenos="18 "></span></a>#com.cloudera.cmf.db.trustCertificateKeyStorePassword=changeit
<a id="__codelineno-9-19" name="__codelineno-9-19"></a><a href="#__codelineno-9-19"><span class="linenos" data-linenos="19 "></span></a>EOF
</code></pre></div>
<ul>
<li>4.5 启动 CM server</li>
</ul>
<div class="highlight"><pre><span></span><code><a id="__codelineno-10-1" name="__codelineno-10-1"></a><a href="#__codelineno-10-1"><span class="linenos" data-linenos="1 "></span></a>systemctl restart cloudera-scm-server
<a id="__codelineno-10-2" name="__codelineno-10-2"></a><a href="#__codelineno-10-2"><span class="linenos" data-linenos="2 "></span></a>tail -f /var/log/cloudera-scm-server/cloudera-scm-server.log
<a id="__codelineno-10-3" name="__codelineno-10-3"></a><a href="#__codelineno-10-3"><span class="linenos" data-linenos="3 "></span></a>
<a id="__codelineno-10-4" name="__codelineno-10-4"></a><a href="#__codelineno-10-4"><span class="linenos" data-linenos="4 "></span></a>systemctl restart cloudera-scm-agent
<a id="__codelineno-10-5" name="__codelineno-10-5"></a><a href="#__codelineno-10-5"><span class="linenos" data-linenos="5 "></span></a>tail -f /var/log/cloudera-scm-server/cloudera-scm-agent.log
</code></pre></div>
<ul>
<li>
<p>4.6 访问 CM</p>
</li>
<li>
<p><a href="http://10.0.0.111:7180/cmf/login">http://10.0.0.111:7180/cmf/login</a>  &nbsp; &nbsp; &nbsp;  默认： <code>admin/admin</code></p>
</li>
</ul>
<h2 id="5-cm">5. 进入 CM 控制台配置集群各组件<a class="headerlink" href="#5-cm" title="Permanent link">&para;</a></h2>
<h3 id="51">5.1 配置截图<a class="headerlink" href="#51" title="Permanent link">&para;</a></h3>
<ul>
<li>
<p>暂略，控制台页面比较简单稍微摸索即可上手，主要管理维度就是：系统设置（添加主机 | parcels 管理 |...）、集群维度 -&gt; 添加角色（就是各种组件）-&gt; 配置（针对所有实例的配置）| 实例（针对单个实例的管理）|...</p>
</li>
<li>
<p>参见1：<a href="https://blog.csdn.net/u010520724/article/details/110635922">blog.csdn.net/u010520724/article/details/110635922</a></p>
</li>
<li>
<p><strong><em>参见2 （有超详细的配置截图）：</em></strong> <a href="https://cloud.tencent.com/developer/article/1079143">cloud.tencent.com/developer/article/1079143</a></p>
</li>
</ul>
<h3 id="52-flink">5.2 集成外部 Flink<a class="headerlink" href="#52-flink" title="Permanent link">&para;</a></h3>
<ul>
<li>参见：<a href="https://gitee.com/wl4g-collect/flink-parcel-generator">gitee.com/wl4g-collect/flink-parcel-generator</a></li>
</ul>
<blockquote>
<p>注：若已集成 Flink parcel 后又想升级版本，则务必注意卸载旧 Flink parcel 的操作顺序：a. 首先在所有引用的集群中停止如 Flink yarn 服务；b. 从每个集群中删除 Flink yarn 角色；c. 再在 parcels 管理中停用 Flink parcel 激活；d. 再点击卸载 Flink parcel</p>
</blockquote>
<h3 id="53-phoenix">5.3 集成外部 Phoenix<a class="headerlink" href="#53-phoenix" title="Permanent link">&para;</a></h3>
<ul>
<li>
<p>参考1：<a href="https://docs.cloudera.com/documentation/enterprise/6/6.3/topics/phoenix_installation.html">docs.cloudera.com/documentation/enterprise/6/6.3/topics/phoenix_installation.html</a></p>
</li>
<li>
<p><a href="https://cloud.tencent.com/developer/article/1078068">（旧，仅参考）CDH-5.x 集成 Phoenix-4.7</a></p>
</li>
</ul>
<blockquote>
<p>提示1：Phoenix 简化了HBase 很多场景的使用方式，在 Cloudera Labs 的其他工具一样，Cloudera 官方目前不会提供Support，仅供实验使用。</p>
<p>提示2：暂时为了简单快速集成，操作步骤：a. 直接从官网下载对应 HBase 的 Phoenix 版本；b. 然后拷贝到所有机器的 CDH HBase 安装目录 lib 下；c. 再重启 HBase 这样就完成了集成。 后续有时间整理打成 phoenix parcel 包标准的方式集成。</p>
</blockquote>
<ul>
<li>如下在 <code>cdh6-master-1</code> 上下载及部署操作示例：</li>
</ul>
<blockquote>
<p>注：一定要找对版本否则会有你想不到的惊喜:)，目前测试发现：<code>cdh6.3.1-hbase-2.1.0</code> + <code>phoenix-hbase-2.1-5.1.2-bin</code>，在首次启动 sqlline.py 时 Phoenix 初始化表会导致 DataNode 报奇怪的错误：<code>DataXceiver error processing WRITE_BLOCK operation IOException: Premature EOF from inputStream</code>，但按此方案解决无效: https://cloud.tencent.com/developer/article/1404118</p>
</blockquote>
<div class="highlight"><pre><span></span><code><a id="__codelineno-11-1" name="__codelineno-11-1"></a><a href="#__codelineno-11-1"><span class="linenos" data-linenos=" 1 "></span></a><span class="nb">cd</span> /opt/cloudera/parcels/
<a id="__codelineno-11-2" name="__codelineno-11-2"></a><a href="#__codelineno-11-2"><span class="linenos" data-linenos=" 2 "></span></a>
<a id="__codelineno-11-3" name="__codelineno-11-3"></a><a href="#__codelineno-11-3"><span class="linenos" data-linenos=" 3 "></span></a><span class="c1">#curl -sSkL -O https://archive.apache.org/dist/phoenix/phoenix-5.1.2/phoenix-hbase-2.1-5.1.2-bin.tar.gz</span>
<a id="__codelineno-11-4" name="__codelineno-11-4"></a><a href="#__codelineno-11-4"><span class="linenos" data-linenos=" 4 "></span></a>curl -sSkL -O http://archive.apache.org/dist/phoenix/phoenix-5.1.1/phoenix-hbase-2.1-5.1.1-bin.tar.gz
<a id="__codelineno-11-5" name="__codelineno-11-5"></a><a href="#__codelineno-11-5"><span class="linenos" data-linenos=" 5 "></span></a>
<a id="__codelineno-11-6" name="__codelineno-11-6"></a><a href="#__codelineno-11-6"><span class="linenos" data-linenos=" 6 "></span></a>tar -xf phoenix-*
<a id="__codelineno-11-7" name="__codelineno-11-7"></a><a href="#__codelineno-11-7"><span class="linenos" data-linenos=" 7 "></span></a>
<a id="__codelineno-11-8" name="__codelineno-11-8"></a><a href="#__codelineno-11-8"><span class="linenos" data-linenos=" 8 "></span></a><span class="c1"># 如下所有 link 操作，所有节点都需要</span>
<a id="__codelineno-11-9" name="__codelineno-11-9"></a><a href="#__codelineno-11-9"><span class="linenos" data-linenos=" 9 "></span></a>ln -snf phoenix-hbase-2.1-5.1.0-bin phoenix
<a id="__codelineno-11-10" name="__codelineno-11-10"></a><a href="#__codelineno-11-10"><span class="linenos" data-linenos="10 "></span></a>
<a id="__codelineno-11-11" name="__codelineno-11-11"></a><a href="#__codelineno-11-11"><span class="linenos" data-linenos="11 "></span></a><span class="c1"># 使 HBase server 自动加载 Phoenix queryserver</span>
<a id="__codelineno-11-12" name="__codelineno-11-12"></a><a href="#__codelineno-11-12"><span class="linenos" data-linenos="12 "></span></a>ln -snf /opt/cloudera/parcels/phoenix/phoenix-server-hbase-2.1-5.1.0.jar /opt/cloudera/parcels/CDH/lib/hbase/lib/phoenix-server-hbase-2.1-5.1.0.jar
</code></pre></div>
<ul>
<li>（<strong>可选</strong>）若您想启用WAL二级索引，则需在重启HBase服务之前，在 <code>hbase-site.xml</code> 中添加以下内容：</li>
</ul>
<div class="highlight"><pre><span></span><code><a id="__codelineno-12-1" name="__codelineno-12-1"></a><a href="#__codelineno-12-1"><span class="linenos" data-linenos="1 "></span></a><span class="nt">&lt;property&gt;</span>
<a id="__codelineno-12-2" name="__codelineno-12-2"></a><a href="#__codelineno-12-2"><span class="linenos" data-linenos="2 "></span></a>    <span class="nt">&lt;name&gt;</span>hbase.regionserver.wal.codec<span class="nt">&lt;/name&gt;</span>
<a id="__codelineno-12-3" name="__codelineno-12-3"></a><a href="#__codelineno-12-3"><span class="linenos" data-linenos="3 "></span></a>    <span class="nt">&lt;value&gt;</span>org.apache.hadoop.hbase.regionserver.wal.IndexedWALEditCodec<span class="nt">&lt;/value&gt;</span>
<a id="__codelineno-12-4" name="__codelineno-12-4"></a><a href="#__codelineno-12-4"><span class="linenos" data-linenos="4 "></span></a><span class="nt">&lt;/property&gt;</span>
</code></pre></div>
<h2 id="6">6. 测试验证<a class="headerlink" href="#6" title="Permanent link">&para;</a></h2>
<h3 id="61-hdfs">6.1 验证 HDFS 部署<a class="headerlink" href="#61-hdfs" title="Permanent link">&para;</a></h3>
<blockquote>
<p>查看 NameNode UI 或当前 NN 进程的配置：http://cdh6-master-1:9870/conf  CDH 默认端口<code>9870</code>，其他工具搭建的集群可能是 <code>50070</code> 等</p>
</blockquote>
<ul>
<li>
<p>前置条件1：首先在 CM 界面配置(调优) HDFS 相关参数，点击“保存更改”，然后在点击 “操作” -&gt; “部署到客户端”，才会在每台主机上生成 file 形式的配置文件，如：<code>cat /etc/hadoop/conf/core-site.xml</code>，否则执行 hdfs 命令时会找不到配置文件。</p>
</li>
<li>
<p>前置条件2：已启动完成 HDFS (NameNode/DataNode/Jouralnode/Httpfs等各进程)</p>
</li>
<li>
<p>执行命令查看文件 HDFS 根目录的文件列表来验证：<code>hdfs dfs -ls /</code></p>
</li>
<li>
<p>总体启动顺序按照如下：
Zookeeper(3) -&gt; HDFS(NameNode/DataNode/Jouralnode/Httpfs..) -&gt; Yarn/mr2(ResourceManager/NodeManager) -&gt; HBase(HMaster/HRegionServer/ThirftServer) -&gt; Kafka(Broker)  -&gt;  Spark(HistoryServer)、Flink(flink-yarn) -&gt; Hue(Hue Server) 等。</p>
</li>
</ul>
<h2 id="62-yarn">6.2 验证 Yarn 部署<a class="headerlink" href="#62-yarn" title="Permanent link">&para;</a></h2>
<blockquote>
<p>查看 Yarn UI 或当前 ResourceManager 进程的配置：http://cdh6-master-1:8088/conf  默认端口<code>8088</code></p>
</blockquote>
<ul>
<li>制作测试数据集并上传到 HDFS</li>
</ul>
<div class="highlight"><pre><span></span><code><a id="__codelineno-13-1" name="__codelineno-13-1"></a><a href="#__codelineno-13-1"><span class="linenos" data-linenos=" 1 "></span></a>mkdir -p /tmp/wordcount/
<a id="__codelineno-13-2" name="__codelineno-13-2"></a><a href="#__codelineno-13-2"><span class="linenos" data-linenos=" 2 "></span></a>cat <span class="s">&lt;&lt;EOF &gt;/tmp/wordcount/hello.txt</span>
<a id="__codelineno-13-3" name="__codelineno-13-3"></a><a href="#__codelineno-13-3"><span class="linenos" data-linenos=" 3 "></span></a><span class="s">hello xm</span>
<a id="__codelineno-13-4" name="__codelineno-13-4"></a><a href="#__codelineno-13-4"><span class="linenos" data-linenos=" 4 "></span></a><span class="s">hello sir</span>
<a id="__codelineno-13-5" name="__codelineno-13-5"></a><a href="#__codelineno-13-5"><span class="linenos" data-linenos=" 5 "></span></a><span class="s">java c</span>
<a id="__codelineno-13-6" name="__codelineno-13-6"></a><a href="#__codelineno-13-6"><span class="linenos" data-linenos=" 6 "></span></a><span class="s">python vb</span>
<a id="__codelineno-13-7" name="__codelineno-13-7"></a><a href="#__codelineno-13-7"><span class="linenos" data-linenos=" 7 "></span></a><span class="s">java c++</span>
<a id="__codelineno-13-8" name="__codelineno-13-8"></a><a href="#__codelineno-13-8"><span class="linenos" data-linenos=" 8 "></span></a><span class="s">go php</span>
<a id="__codelineno-13-9" name="__codelineno-13-9"></a><a href="#__codelineno-13-9"><span class="linenos" data-linenos=" 9 "></span></a><span class="s">erlang java</span>
<a id="__codelineno-13-10" name="__codelineno-13-10"></a><a href="#__codelineno-13-10"><span class="linenos" data-linenos="10 "></span></a><span class="s">EOF</span>
<a id="__codelineno-13-11" name="__codelineno-13-11"></a><a href="#__codelineno-13-11"><span class="linenos" data-linenos="11 "></span></a>
<a id="__codelineno-13-12" name="__codelineno-13-12"></a><a href="#__codelineno-13-12"><span class="linenos" data-linenos="12 "></span></a>sudo -u hdfs hdfs dfs -mkdir -p /tmp/
<a id="__codelineno-13-13" name="__codelineno-13-13"></a><a href="#__codelineno-13-13"><span class="linenos" data-linenos="13 "></span></a>sudo -u hdfs hdfs dfs -put /tmp/wordcount/ /tmp
</code></pre></div>
<ul>
<li>启动 MR 任务</li>
</ul>
<blockquote>
<p>注：启动之前必须在 CM 控制台上点击执行安装新MR框架 jar，菜单在：“集群“ -&gt; ”“YARN (MR2 Included)” -&gt; “安装 YARN MapReduce 框架 JAR 命令”，由于 Yarn 的前身 V1 是 MapReduce ，后来 V2 升级为 Yarn。see：<a href="https://docs.cloudera.com/documentation/enterprise/6/6.3/topics/cm_mc_mr_and_yarn.html">docs.cloudera.com/documentation/enterprise/6/6.3/topics/cm_mc_mr_and_yarn.html</a></p>
</blockquote>
<ul>
<li>从命令日志看就是执行的这两条命令：</li>
</ul>
<div class="highlight"><pre><span></span><code><a id="__codelineno-14-1" name="__codelineno-14-1"></a><a href="#__codelineno-14-1"><span class="linenos" data-linenos="1 "></span></a>hdfs/hdfs.sh <span class="o">[</span><span class="s2">&quot;mkdir&quot;</span>,<span class="s2">&quot;/user/yarn/mapreduce/mr-framework&quot;</span>,<span class="s2">&quot;yarn&quot;</span>,<span class="s2">&quot;hadoop&quot;</span>,<span class="s2">&quot;775&quot;</span><span class="o">]</span><span class="sb">`</span>
<a id="__codelineno-14-2" name="__codelineno-14-2"></a><a href="#__codelineno-14-2"><span class="linenos" data-linenos="2 "></span></a>yarn/yarn.sh <span class="o">[</span><span class="s2">&quot;install-mr-framework&quot;</span>,<span class="s2">&quot;/user/yarn/mapreduce/mr-framework/3.0.0-cdh6.3.1-mr-framework.tar.gz#mr-framework&quot;</span>,<span class="s2">&quot;hdfs://cdh6-master-1:8020&quot;</span><span class="o">]</span>
</code></pre></div>
<div class="highlight"><pre><span></span><code><a id="__codelineno-15-1" name="__codelineno-15-1"></a><a href="#__codelineno-15-1"><span class="linenos" data-linenos=" 1 "></span></a><span class="nb">cd</span> /opt/cloudera/parcels/CDH-6.3.1-1.cdh6.3.1.p0.1470567/lib/hadoop-mapreduce/
<a id="__codelineno-15-2" name="__codelineno-15-2"></a><a href="#__codelineno-15-2"><span class="linenos" data-linenos=" 2 "></span></a>
<a id="__codelineno-15-3" name="__codelineno-15-3"></a><a href="#__codelineno-15-3"><span class="linenos" data-linenos=" 3 "></span></a><span class="c1"># 【注】：执行用户(sudo -u hdfs)，否则可能报错：Cleaning up the staging area file:/tmp/hadoop/mapred/staging/hdfs1449449357/.staging/job_local1449449357_0001 ENOENT: No such file or directory</span>
<a id="__codelineno-15-4" name="__codelineno-15-4"></a><a href="#__codelineno-15-4"><span class="linenos" data-linenos=" 4 "></span></a>sudo -u hdfs yarn jar hadoop-mapreduce-examples-3.0.0-cdh6.3.1.jar wordcount /tmp/wordcount/hello.txt /tmp/wordcount/output
<a id="__codelineno-15-5" name="__codelineno-15-5"></a><a href="#__codelineno-15-5"><span class="linenos" data-linenos=" 5 "></span></a>
<a id="__codelineno-15-6" name="__codelineno-15-6"></a><a href="#__codelineno-15-6"><span class="linenos" data-linenos=" 6 "></span></a><span class="c1"># 查看 yarn 任务容器</span>
<a id="__codelineno-15-7" name="__codelineno-15-7"></a><a href="#__codelineno-15-7"><span class="linenos" data-linenos=" 7 "></span></a>yarn application -list
<a id="__codelineno-15-8" name="__codelineno-15-8"></a><a href="#__codelineno-15-8"><span class="linenos" data-linenos=" 8 "></span></a>
<a id="__codelineno-15-9" name="__codelineno-15-9"></a><a href="#__codelineno-15-9"><span class="linenos" data-linenos=" 9 "></span></a><span class="c1"># 查看结果集</span>
<a id="__codelineno-15-10" name="__codelineno-15-10"></a><a href="#__codelineno-15-10"><span class="linenos" data-linenos="10 "></span></a>hdfs dfs -ls /tmp/wordcount/output/
<a id="__codelineno-15-11" name="__codelineno-15-11"></a><a href="#__codelineno-15-11"><span class="linenos" data-linenos="11 "></span></a>hdfs dfs -cat /tmp/wordcount/output/part-r-*
</code></pre></div>
<h2 id="63-hbase">6.3 验证 HBase 部署<a class="headerlink" href="#63-hbase" title="Permanent link">&para;</a></h2>
<blockquote>
<p>查看 HMaster UI 或当前 HMaster 进程的配置：http://cdh6-master-1:16010/conf  默认端口<code>16010</code></p>
</blockquote>
<ul>
<li>使用 hbase shell 建表写数据</li>
</ul>
<div class="highlight"><pre><span></span><code><a id="__codelineno-16-1" name="__codelineno-16-1"></a><a href="#__codelineno-16-1"><span class="linenos" data-linenos=" 1 "></span></a>hbase shell
<a id="__codelineno-16-2" name="__codelineno-16-2"></a><a href="#__codelineno-16-2"><span class="linenos" data-linenos=" 2 "></span></a>
<a id="__codelineno-16-3" name="__codelineno-16-3"></a><a href="#__codelineno-16-3"><span class="linenos" data-linenos=" 3 "></span></a>hbase<span class="o">(</span>main<span class="o">)</span>:005:0&gt; version
<a id="__codelineno-16-4" name="__codelineno-16-4"></a><a href="#__codelineno-16-4"><span class="linenos" data-linenos=" 4 "></span></a><span class="m">2</span>.1.0-cdh6.3.1, rUnknown, Thu Sep <span class="m">26</span> <span class="m">02</span>:56:37 PDT <span class="m">2019</span>
<a id="__codelineno-16-5" name="__codelineno-16-5"></a><a href="#__codelineno-16-5"><span class="linenos" data-linenos=" 5 "></span></a>Took <span class="m">0</span>.0007 seconds
<a id="__codelineno-16-6" name="__codelineno-16-6"></a><a href="#__codelineno-16-6"><span class="linenos" data-linenos=" 6 "></span></a>
<a id="__codelineno-16-7" name="__codelineno-16-7"></a><a href="#__codelineno-16-7"><span class="linenos" data-linenos=" 7 "></span></a>hbase<span class="o">(</span>main<span class="o">)</span>:006:0&gt; status
<a id="__codelineno-16-8" name="__codelineno-16-8"></a><a href="#__codelineno-16-8"><span class="linenos" data-linenos=" 8 "></span></a><span class="m">1</span> active master, <span class="m">0</span> backup masters, <span class="m">2</span> servers, <span class="m">0</span> dead, <span class="m">1</span>.0000 average load
<a id="__codelineno-16-9" name="__codelineno-16-9"></a><a href="#__codelineno-16-9"><span class="linenos" data-linenos=" 9 "></span></a>Took <span class="m">0</span>.0186 seconds
<a id="__codelineno-16-10" name="__codelineno-16-10"></a><a href="#__codelineno-16-10"><span class="linenos" data-linenos="10 "></span></a>
<a id="__codelineno-16-11" name="__codelineno-16-11"></a><a href="#__codelineno-16-11"><span class="linenos" data-linenos="11 "></span></a><span class="c1"># 创建带有2个列簇的表</span>
<a id="__codelineno-16-12" name="__codelineno-16-12"></a><a href="#__codelineno-16-12"><span class="linenos" data-linenos="12 "></span></a>hbase<span class="o">(</span>main<span class="o">)</span>:008:0&gt; create <span class="s1">&#39;t_hello&#39;</span>, <span class="o">{</span><span class="nv">NAME</span><span class="o">=</span>&gt;<span class="s1">&#39;f1&#39;</span><span class="o">}</span>, <span class="o">{</span><span class="nv">NAME</span><span class="o">=</span>&gt;<span class="s1">&#39;f2&#39;</span><span class="o">}</span>
<a id="__codelineno-16-13" name="__codelineno-16-13"></a><a href="#__codelineno-16-13"><span class="linenos" data-linenos="13 "></span></a>Created table t_hello
<a id="__codelineno-16-14" name="__codelineno-16-14"></a><a href="#__codelineno-16-14"><span class="linenos" data-linenos="14 "></span></a>Took <span class="m">2</span>.9259 <span class="nv">seconds</span>
<a id="__codelineno-16-15" name="__codelineno-16-15"></a><a href="#__codelineno-16-15"><span class="linenos" data-linenos="15 "></span></a><span class="o">=</span>&gt; Hbase::Table - t_hello
<a id="__codelineno-16-16" name="__codelineno-16-16"></a><a href="#__codelineno-16-16"><span class="linenos" data-linenos="16 "></span></a>
<a id="__codelineno-16-17" name="__codelineno-16-17"></a><a href="#__codelineno-16-17"><span class="linenos" data-linenos="17 "></span></a><span class="c1"># 列出所有表</span>
<a id="__codelineno-16-18" name="__codelineno-16-18"></a><a href="#__codelineno-16-18"><span class="linenos" data-linenos="18 "></span></a>hbase<span class="o">(</span>main<span class="o">)</span>:016:0&gt; list
<a id="__codelineno-16-19" name="__codelineno-16-19"></a><a href="#__codelineno-16-19"><span class="linenos" data-linenos="19 "></span></a>TABLE
<a id="__codelineno-16-20" name="__codelineno-16-20"></a><a href="#__codelineno-16-20"><span class="linenos" data-linenos="20 "></span></a>t_hello
<a id="__codelineno-16-21" name="__codelineno-16-21"></a><a href="#__codelineno-16-21"><span class="linenos" data-linenos="21 "></span></a><span class="m">1</span> row<span class="o">(</span>s<span class="o">)</span>
<a id="__codelineno-16-22" name="__codelineno-16-22"></a><a href="#__codelineno-16-22"><span class="linenos" data-linenos="22 "></span></a>Took <span class="m">0</span>.0181 <span class="nv">seconds</span>
<a id="__codelineno-16-23" name="__codelineno-16-23"></a><a href="#__codelineno-16-23"><span class="linenos" data-linenos="23 "></span></a><span class="o">=</span>&gt; <span class="o">[</span><span class="s2">&quot;t_hello&quot;</span><span class="o">]</span>
<a id="__codelineno-16-24" name="__codelineno-16-24"></a><a href="#__codelineno-16-24"><span class="linenos" data-linenos="24 "></span></a>
<a id="__codelineno-16-25" name="__codelineno-16-25"></a><a href="#__codelineno-16-25"><span class="linenos" data-linenos="25 "></span></a><span class="c1"># 查看表结构</span>
<a id="__codelineno-16-26" name="__codelineno-16-26"></a><a href="#__codelineno-16-26"><span class="linenos" data-linenos="26 "></span></a>hbase<span class="o">(</span>main<span class="o">)</span>:010:0&gt; describe <span class="s1">&#39;t_hello&#39;</span>
<a id="__codelineno-16-27" name="__codelineno-16-27"></a><a href="#__codelineno-16-27"><span class="linenos" data-linenos="27 "></span></a>Table t_hello is ENABLED
<a id="__codelineno-16-28" name="__codelineno-16-28"></a><a href="#__codelineno-16-28"><span class="linenos" data-linenos="28 "></span></a>t_hello
<a id="__codelineno-16-29" name="__codelineno-16-29"></a><a href="#__codelineno-16-29"><span class="linenos" data-linenos="29 "></span></a>COLUMN FAMILIES DESCRIPTION
<a id="__codelineno-16-30" name="__codelineno-16-30"></a><a href="#__codelineno-16-30"><span class="linenos" data-linenos="30 "></span></a><span class="o">{</span><span class="nv">NAME</span> <span class="o">=</span>&gt; <span class="s1">&#39;f1&#39;</span>, <span class="nv">VERSIONS</span> <span class="o">=</span>&gt; <span class="s1">&#39;1&#39;</span>, <span class="nv">EVICT_BLOCKS_ON_CLOSE</span> <span class="o">=</span>&gt; <span class="s1">&#39;false&#39;</span>, <span class="nv">NEW_VERSION_BEHAVIOR</span> <span class="o">=</span>&gt; <span class="s1">&#39;false&#39;</span>, <span class="nv">KEEP_DELETED_CELLS</span> <span class="o">=</span>&gt; <span class="s1">&#39;FALSE&#39;</span>, <span class="nv">CACHE_DATA_ON_WRITE</span> <span class="o">=</span>&gt; <span class="s1">&#39;false&#39;</span>, 
<a id="__codelineno-16-31" name="__codelineno-16-31"></a><a href="#__codelineno-16-31"><span class="linenos" data-linenos="31 "></span></a><span class="nv">DATA_BLOCK_ENCODING</span> <span class="o">=</span>&gt; <span class="s1">&#39;NONE&#39;</span>, <span class="nv">TTL</span> <span class="o">=</span>&gt; <span class="s1">&#39;FOREVER&#39;</span>, <span class="nv">MIN_VERSIONS</span> <span class="o">=</span>&gt; <span class="s1">&#39;0&#39;</span>, <span class="nv">REPLICATION_SCOPE</span> <span class="o">=</span>&gt; <span class="s1">&#39;0&#39;</span>, <span class="nv">BLOOMFILTER</span> <span class="o">=</span>&gt; <span class="s1">&#39;ROW&#39;</span>, <span class="nv">CACHE_INDEX_ON_WRITE</span> <span class="o">=</span>&gt; <span class="s1">&#39;false&#39;</span>, <span class="nv">IN_MEMORY</span> <span class="o">=</span>
<a id="__codelineno-16-32" name="__codelineno-16-32"></a><a href="#__codelineno-16-32"><span class="linenos" data-linenos="32 "></span></a>&gt; <span class="s1">&#39;false&#39;</span>, <span class="nv">CACHE_BLOOMS_ON_WRITE</span> <span class="o">=</span>&gt; <span class="s1">&#39;false&#39;</span>, <span class="nv">PREFETCH_BLOCKS_ON_OPEN</span> <span class="o">=</span>&gt; <span class="s1">&#39;false&#39;</span>, <span class="nv">COMPRESSION</span> <span class="o">=</span>&gt; <span class="s1">&#39;NONE&#39;</span>, <span class="nv">BLOCKCACHE</span> <span class="o">=</span>&gt; <span class="s1">&#39;true&#39;</span>, <span class="nv">BLOCKSIZE</span> <span class="o">=</span>&gt; <span class="s1">&#39;65536&#39;</span><span class="o">}</span>
<a id="__codelineno-16-33" name="__codelineno-16-33"></a><a href="#__codelineno-16-33"><span class="linenos" data-linenos="33 "></span></a><span class="o">{</span><span class="nv">NAME</span> <span class="o">=</span>&gt; <span class="s1">&#39;f2&#39;</span>, <span class="nv">VERSIONS</span> <span class="o">=</span>&gt; <span class="s1">&#39;1&#39;</span>, <span class="nv">EVICT_BLOCKS_ON_CLOSE</span> <span class="o">=</span>&gt; <span class="s1">&#39;false&#39;</span>, <span class="nv">NEW_VERSION_BEHAVIOR</span> <span class="o">=</span>&gt; <span class="s1">&#39;false&#39;</span>, <span class="nv">KEEP_DELETED_CELLS</span> <span class="o">=</span>&gt; <span class="s1">&#39;FALSE&#39;</span>, <span class="nv">CACHE_DATA_ON_WRITE</span> <span class="o">=</span>&gt; <span class="s1">&#39;false&#39;</span>, 
<a id="__codelineno-16-34" name="__codelineno-16-34"></a><a href="#__codelineno-16-34"><span class="linenos" data-linenos="34 "></span></a><span class="nv">DATA_BLOCK_ENCODING</span> <span class="o">=</span>&gt; <span class="s1">&#39;NONE&#39;</span>, <span class="nv">TTL</span> <span class="o">=</span>&gt; <span class="s1">&#39;FOREVER&#39;</span>, <span class="nv">MIN_VERSIONS</span> <span class="o">=</span>&gt; <span class="s1">&#39;0&#39;</span>, <span class="nv">REPLICATION_SCOPE</span> <span class="o">=</span>&gt; <span class="s1">&#39;0&#39;</span>, <span class="nv">BLOOMFILTER</span> <span class="o">=</span>&gt; <span class="s1">&#39;ROW&#39;</span>, <span class="nv">CACHE_INDEX_ON_WRITE</span> <span class="o">=</span>&gt; <span class="s1">&#39;false&#39;</span>, <span class="nv">IN_MEMORY</span> <span class="o">=</span>
<a id="__codelineno-16-35" name="__codelineno-16-35"></a><a href="#__codelineno-16-35"><span class="linenos" data-linenos="35 "></span></a>&gt; <span class="s1">&#39;false&#39;</span>, <span class="nv">CACHE_BLOOMS_ON_WRITE</span> <span class="o">=</span>&gt; <span class="s1">&#39;false&#39;</span>, <span class="nv">PREFETCH_BLOCKS_ON_OPEN</span> <span class="o">=</span>&gt; <span class="s1">&#39;false&#39;</span>, <span class="nv">COMPRESSION</span> <span class="o">=</span>&gt; <span class="s1">&#39;NONE&#39;</span>, <span class="nv">BLOCKCACHE</span> <span class="o">=</span>&gt; <span class="s1">&#39;true&#39;</span>, <span class="nv">BLOCKSIZE</span> <span class="o">=</span>&gt; <span class="s1">&#39;65536&#39;</span><span class="o">}</span>
<a id="__codelineno-16-36" name="__codelineno-16-36"></a><a href="#__codelineno-16-36"><span class="linenos" data-linenos="36 "></span></a><span class="m">2</span> row<span class="o">(</span>s<span class="o">)</span>
<a id="__codelineno-16-37" name="__codelineno-16-37"></a><a href="#__codelineno-16-37"><span class="linenos" data-linenos="37 "></span></a>Took <span class="m">0</span>.2506 seconds
<a id="__codelineno-16-38" name="__codelineno-16-38"></a><a href="#__codelineno-16-38"><span class="linenos" data-linenos="38 "></span></a>
<a id="__codelineno-16-39" name="__codelineno-16-39"></a><a href="#__codelineno-16-39"><span class="linenos" data-linenos="39 "></span></a><span class="c1"># 向表添加数据</span>
<a id="__codelineno-16-40" name="__codelineno-16-40"></a><a href="#__codelineno-16-40"><span class="linenos" data-linenos="40 "></span></a>hbase<span class="o">(</span>main<span class="o">)</span>:012:0&gt; put <span class="s1">&#39;t_hello&#39;</span>, <span class="s1">&#39;001&#39;</span>,<span class="s1">&#39;f1:name&#39;</span>,<span class="s1">&#39;Tom&#39;</span>
<a id="__codelineno-16-41" name="__codelineno-16-41"></a><a href="#__codelineno-16-41"><span class="linenos" data-linenos="41 "></span></a>Took <span class="m">0</span>.1881 seconds
<a id="__codelineno-16-42" name="__codelineno-16-42"></a><a href="#__codelineno-16-42"><span class="linenos" data-linenos="42 "></span></a>hbase<span class="o">(</span>main<span class="o">)</span>:013:0&gt; put <span class="s1">&#39;t_hello&#39;</span>, <span class="s1">&#39;001&#39;</span>,<span class="s1">&#39;f1:gender&#39;</span>,<span class="s1">&#39;Man&#39;</span>
<a id="__codelineno-16-43" name="__codelineno-16-43"></a><a href="#__codelineno-16-43"><span class="linenos" data-linenos="43 "></span></a>Took <span class="m">0</span>.0201 seconds
<a id="__codelineno-16-44" name="__codelineno-16-44"></a><a href="#__codelineno-16-44"><span class="linenos" data-linenos="44 "></span></a>hbase<span class="o">(</span>main<span class="o">)</span>:014:0&gt; put <span class="s1">&#39;t_hello&#39;</span>, <span class="s1">&#39;001&#39;</span>,<span class="s1">&#39;f1:lang&#39;</span>,<span class="s1">&#39;CN&#39;</span>
<a id="__codelineno-16-45" name="__codelineno-16-45"></a><a href="#__codelineno-16-45"><span class="linenos" data-linenos="45 "></span></a>Took <span class="m">0</span>.0196 seconds
<a id="__codelineno-16-46" name="__codelineno-16-46"></a><a href="#__codelineno-16-46"><span class="linenos" data-linenos="46 "></span></a>
<a id="__codelineno-16-47" name="__codelineno-16-47"></a><a href="#__codelineno-16-47"><span class="linenos" data-linenos="47 "></span></a><span class="c1"># 扫描查询表数据</span>
<a id="__codelineno-16-48" name="__codelineno-16-48"></a><a href="#__codelineno-16-48"><span class="linenos" data-linenos="48 "></span></a>hbase<span class="o">(</span>main<span class="o">)</span>:015:0&gt; scan <span class="s1">&#39;t_hello&#39;</span>
<a id="__codelineno-16-49" name="__codelineno-16-49"></a><a href="#__codelineno-16-49"><span class="linenos" data-linenos="49 "></span></a>ROW                                       COLUMN+CELL
<a id="__codelineno-16-50" name="__codelineno-16-50"></a><a href="#__codelineno-16-50"><span class="linenos" data-linenos="50 "></span></a> <span class="m">001</span>                                      <span class="nv">column</span><span class="o">=</span>f1:gender, <span class="nv">timestamp</span><span class="o">=</span><span class="m">1653460232218</span>, <span class="nv">value</span><span class="o">=</span>Man
<a id="__codelineno-16-51" name="__codelineno-16-51"></a><a href="#__codelineno-16-51"><span class="linenos" data-linenos="51 "></span></a> <span class="m">001</span>                                      <span class="nv">column</span><span class="o">=</span>f1:lang, <span class="nv">timestamp</span><span class="o">=</span><span class="m">1653460248593</span>, <span class="nv">value</span><span class="o">=</span>CN
<a id="__codelineno-16-52" name="__codelineno-16-52"></a><a href="#__codelineno-16-52"><span class="linenos" data-linenos="52 "></span></a> <span class="m">001</span>                                      <span class="nv">column</span><span class="o">=</span>f1:name, <span class="nv">timestamp</span><span class="o">=</span><span class="m">1653460206910</span>, <span class="nv">value</span><span class="o">=</span>Tom
<a id="__codelineno-16-53" name="__codelineno-16-53"></a><a href="#__codelineno-16-53"><span class="linenos" data-linenos="53 "></span></a><span class="m">1</span> row<span class="o">(</span>s<span class="o">)</span>
<a id="__codelineno-16-54" name="__codelineno-16-54"></a><a href="#__codelineno-16-54"><span class="linenos" data-linenos="54 "></span></a>Took <span class="m">0</span>.1212 seconds
</code></pre></div>
<h2 id="64-spark">6.4 验证 Spark 部署<a class="headerlink" href="#64-spark" title="Permanent link">&para;</a></h2>
<ul>
<li><a href="https://spark.apache.org/docs/2.4.0/quick-start.html">https://spark.apache.org/docs/2.4.0/quick-start.html</a></li>
</ul>
<blockquote>
<p>查看 Spark UI：http://cdh6-master-1:4040/  Standalone模式默认端口<code>4040</code>，注：若使用 yarn 调度，则每个 SparkSubmit 任务UI 地址不同，应从 yarn 控制台 RUNNING 列表跳转过来查看。</p>
</blockquote>
<div class="highlight"><pre><span></span><code><a id="__codelineno-17-1" name="__codelineno-17-1"></a><a href="#__codelineno-17-1"><span class="linenos" data-linenos=" 1 "></span></a><span class="c1"># 沿用 7.1 的 wordcount 数据集</span>
<a id="__codelineno-17-2" name="__codelineno-17-2"></a><a href="#__codelineno-17-2"><span class="linenos" data-linenos=" 2 "></span></a>spark-shell
<a id="__codelineno-17-3" name="__codelineno-17-3"></a><a href="#__codelineno-17-3"><span class="linenos" data-linenos=" 3 "></span></a>
<a id="__codelineno-17-4" name="__codelineno-17-4"></a><a href="#__codelineno-17-4"><span class="linenos" data-linenos=" 4 "></span></a>scala&gt; val <span class="nv">data</span><span class="o">=</span>sc.textFile<span class="o">(</span><span class="s2">&quot;/tmp/wordcount/hello.txt&quot;</span><span class="o">)</span>
<a id="__codelineno-17-5" name="__codelineno-17-5"></a><a href="#__codelineno-17-5"><span class="linenos" data-linenos=" 5 "></span></a>data: org.apache.spark.rdd.RDD<span class="o">[</span>String<span class="o">]</span> <span class="o">=</span> /tmp/wordcount/hello.txt MapPartitionsRDD<span class="o">[</span><span class="m">1</span><span class="o">]</span> at textFile at &lt;console&gt;:24
<a id="__codelineno-17-6" name="__codelineno-17-6"></a><a href="#__codelineno-17-6"><span class="linenos" data-linenos=" 6 "></span></a>
<a id="__codelineno-17-7" name="__codelineno-17-7"></a><a href="#__codelineno-17-7"><span class="linenos" data-linenos=" 7 "></span></a>scala&gt; val <span class="nv">textFile</span> <span class="o">=</span> spark.read.textFile<span class="o">(</span><span class="s2">&quot;/tmp/wordcount/hello.txt&quot;</span><span class="o">)</span>
<a id="__codelineno-17-8" name="__codelineno-17-8"></a><a href="#__codelineno-17-8"><span class="linenos" data-linenos=" 8 "></span></a>textFile: org.apache.spark.sql.Dataset<span class="o">[</span>String<span class="o">]</span> <span class="o">=</span> <span class="o">[</span>value: string<span class="o">]</span>
<a id="__codelineno-17-9" name="__codelineno-17-9"></a><a href="#__codelineno-17-9"><span class="linenos" data-linenos=" 9 "></span></a>
<a id="__codelineno-17-10" name="__codelineno-17-10"></a><a href="#__codelineno-17-10"><span class="linenos" data-linenos="10 "></span></a>scala&gt; data.collect<span class="p">;</span>
<a id="__codelineno-17-11" name="__codelineno-17-11"></a><a href="#__codelineno-17-11"><span class="linenos" data-linenos="11 "></span></a>res0: Array<span class="o">[</span>String<span class="o">]</span> <span class="o">=</span> Array<span class="o">(</span>hello xm, hello sir, java c, python vb, java c++, go php, erlang java<span class="o">)</span>
<a id="__codelineno-17-12" name="__codelineno-17-12"></a><a href="#__codelineno-17-12"><span class="linenos" data-linenos="12 "></span></a>
<a id="__codelineno-17-13" name="__codelineno-17-13"></a><a href="#__codelineno-17-13"><span class="linenos" data-linenos="13 "></span></a>scala&gt; val <span class="nv">splitdata</span> <span class="o">=</span> data.flatMap<span class="o">(</span><span class="nv">line</span> <span class="o">=</span>&gt; line.split<span class="o">(</span><span class="s2">&quot; &quot;</span><span class="o">))</span><span class="p">;</span>
<a id="__codelineno-17-14" name="__codelineno-17-14"></a><a href="#__codelineno-17-14"><span class="linenos" data-linenos="14 "></span></a>splitdata: org.apache.spark.rdd.RDD<span class="o">[</span>String<span class="o">]</span> <span class="o">=</span> MapPartitionsRDD<span class="o">[</span><span class="m">2</span><span class="o">]</span> at flatMap at &lt;console&gt;:25
<a id="__codelineno-17-15" name="__codelineno-17-15"></a><a href="#__codelineno-17-15"><span class="linenos" data-linenos="15 "></span></a>
<a id="__codelineno-17-16" name="__codelineno-17-16"></a><a href="#__codelineno-17-16"><span class="linenos" data-linenos="16 "></span></a>scala&gt; splitdata.collect<span class="p">;</span>
<a id="__codelineno-17-17" name="__codelineno-17-17"></a><a href="#__codelineno-17-17"><span class="linenos" data-linenos="17 "></span></a>res1: Array<span class="o">[</span>String<span class="o">]</span> <span class="o">=</span> Array<span class="o">(</span>hello, xm, hello, sir, java, c, python, vb, java, c++, go, php, erlang, java<span class="o">)</span>
</code></pre></div>
<ul>
<li>示例计算 Pi</li>
</ul>
<div class="highlight"><pre><span></span><code><a id="__codelineno-18-1" name="__codelineno-18-1"></a><a href="#__codelineno-18-1"><span class="linenos" data-linenos="1 "></span></a>sudo -u hdfs spark-submit --class org.apache.spark.examples.SparkPi --master yarn --deploy-mode cluster /opt/cloudera/parcels/CDH/lib/spark/examples/jars/spark-examples_2.11-2.4.0-cdh6.3.1.jar <span class="m">10</span>
</code></pre></div>
<h2 id="65-flink">6.5 验证 Flink 部署<a class="headerlink" href="#65-flink" title="Permanent link">&para;</a></h2>
<ul>
<li><a href="https://nightlies.apache.org/flink/flink-docs-release-1.0/apis/batch/examples.html">nightlies.apache.org/flink/flink-docs-release-1.0/apis/batch/examples.html</a></li>
</ul>
<blockquote>
<p>查看 Flink UI：http://cdh6-master-1:8081/  Standalone模式默认端口<code>8081</code>，注：若使用 yarn 调度，则每个 (TaskManagerRunner / CliFrontend 等) 任务UI 地址不同，应从 yarn 控制台 RUNNING 列表跳转过来查看。</p>
</blockquote>
<ul>
<li>运行官方 WordCount 示例任务：</li>
</ul>
<div class="highlight"><pre><span></span><code><a id="__codelineno-19-1" name="__codelineno-19-1"></a><a href="#__codelineno-19-1"><span class="linenos" data-linenos=" 1 "></span></a><span class="c1"># 还是沿用 7.1 的 wordcount 数据集（不加 &#39;hdfs://&#39; 默认读取 local FS）</span>
<a id="__codelineno-19-2" name="__codelineno-19-2"></a><a href="#__codelineno-19-2"><span class="linenos" data-linenos=" 2 "></span></a><span class="nb">cd</span> /opt/cloudera/parcels/flink-1.14.4-bin-scala_2.11/lib/flink/
<a id="__codelineno-19-3" name="__codelineno-19-3"></a><a href="#__codelineno-19-3"><span class="linenos" data-linenos=" 3 "></span></a>
<a id="__codelineno-19-4" name="__codelineno-19-4"></a><a href="#__codelineno-19-4"><span class="linenos" data-linenos=" 4 "></span></a><span class="c1"># 先local 模式部署：</span>
<a id="__codelineno-19-5" name="__codelineno-19-5"></a><a href="#__codelineno-19-5"><span class="linenos" data-linenos=" 5 "></span></a>sudo -u hdfs ./bin/flink run -t <span class="nb">local</span> examples/batch/WordCount.jar --input hdfs:///tmp/wordcount/hello.txt --output hdfs:///tmp/wordcount/flink-output.txt
<a id="__codelineno-19-6" name="__codelineno-19-6"></a><a href="#__codelineno-19-6"><span class="linenos" data-linenos=" 6 "></span></a>
<a id="__codelineno-19-7" name="__codelineno-19-7"></a><a href="#__codelineno-19-7"><span class="linenos" data-linenos=" 7 "></span></a><span class="c1"># 再使用 yarn-per-job 模式部署</span>
<a id="__codelineno-19-8" name="__codelineno-19-8"></a><a href="#__codelineno-19-8"><span class="linenos" data-linenos=" 8 "></span></a>sudo -u hdfs ./bin/flink run -t yarn-per-job examples/batch/WordCount.jar --input hdfs:///tmp/wordcount/hello.txt --output hdfs:///tmp/wordcount/flink-output.txt
<a id="__codelineno-19-9" name="__codelineno-19-9"></a><a href="#__codelineno-19-9"><span class="linenos" data-linenos=" 9 "></span></a>
<a id="__codelineno-19-10" name="__codelineno-19-10"></a><a href="#__codelineno-19-10"><span class="linenos" data-linenos="10 "></span></a><span class="c1"># 查看结果集</span>
<a id="__codelineno-19-11" name="__codelineno-19-11"></a><a href="#__codelineno-19-11"><span class="linenos" data-linenos="11 "></span></a>hdfs dfs -cat /tmp/wordcount/flink-output.txt
<a id="__codelineno-19-12" name="__codelineno-19-12"></a><a href="#__codelineno-19-12"><span class="linenos" data-linenos="12 "></span></a>c <span class="m">2</span>
<a id="__codelineno-19-13" name="__codelineno-19-13"></a><a href="#__codelineno-19-13"><span class="linenos" data-linenos="13 "></span></a>erlang <span class="m">1</span>
<a id="__codelineno-19-14" name="__codelineno-19-14"></a><a href="#__codelineno-19-14"><span class="linenos" data-linenos="14 "></span></a>go <span class="m">1</span>
<a id="__codelineno-19-15" name="__codelineno-19-15"></a><a href="#__codelineno-19-15"><span class="linenos" data-linenos="15 "></span></a>hello <span class="m">2</span>
<a id="__codelineno-19-16" name="__codelineno-19-16"></a><a href="#__codelineno-19-16"><span class="linenos" data-linenos="16 "></span></a>java <span class="m">3</span>
<a id="__codelineno-19-17" name="__codelineno-19-17"></a><a href="#__codelineno-19-17"><span class="linenos" data-linenos="17 "></span></a>php <span class="m">1</span>
<a id="__codelineno-19-18" name="__codelineno-19-18"></a><a href="#__codelineno-19-18"><span class="linenos" data-linenos="18 "></span></a>python <span class="m">1</span>
<a id="__codelineno-19-19" name="__codelineno-19-19"></a><a href="#__codelineno-19-19"><span class="linenos" data-linenos="19 "></span></a>sir <span class="m">1</span>
<a id="__codelineno-19-20" name="__codelineno-19-20"></a><a href="#__codelineno-19-20"><span class="linenos" data-linenos="20 "></span></a>vb <span class="m">1</span>
<a id="__codelineno-19-21" name="__codelineno-19-21"></a><a href="#__codelineno-19-21"><span class="linenos" data-linenos="21 "></span></a>xm <span class="m">1</span>
</code></pre></div>
<ul>
<li>运行 Flink Shell 交互式任务：</li>
</ul>
<blockquote>
<p><strong>注:</strong> 到 flink-1.14.4为止还不支持 scala2.12, 参见本文 FAQ #8.7</p>
</blockquote>
<div class="highlight"><pre><span></span><code><a id="__codelineno-20-1" name="__codelineno-20-1"></a><a href="#__codelineno-20-1"><span class="linenos" data-linenos="1 "></span></a><span class="nb">cd</span> /opt/cloudera/parcels/flink-1.14.4-bin-scala_2.11/lib/flink/
<a id="__codelineno-20-2" name="__codelineno-20-2"></a><a href="#__codelineno-20-2"><span class="linenos" data-linenos="2 "></span></a>./bin/start-scala-shell.sh <span class="nb">local</span>
<a id="__codelineno-20-3" name="__codelineno-20-3"></a><a href="#__codelineno-20-3"><span class="linenos" data-linenos="3 "></span></a>
<a id="__codelineno-20-4" name="__codelineno-20-4"></a><a href="#__codelineno-20-4"><span class="linenos" data-linenos="4 "></span></a>scala&gt; val <span class="nv">dataStream</span> <span class="o">=</span> senv.fromElements<span class="o">(</span><span class="m">1</span>, <span class="m">2</span>, <span class="m">3</span>, <span class="m">4</span><span class="o">)</span>
<a id="__codelineno-20-5" name="__codelineno-20-5"></a><a href="#__codelineno-20-5"><span class="linenos" data-linenos="5 "></span></a>scala&gt; dataStream.countWindowAll<span class="o">(</span><span class="m">2</span><span class="o">)</span>.sum<span class="o">(</span><span class="m">0</span><span class="o">)</span>.executeAndCollect<span class="o">()</span>.foreach<span class="o">(</span>println<span class="o">)</span>
<a id="__codelineno-20-6" name="__codelineno-20-6"></a><a href="#__codelineno-20-6"><span class="linenos" data-linenos="6 "></span></a><span class="m">3</span>
<a id="__codelineno-20-7" name="__codelineno-20-7"></a><a href="#__codelineno-20-7"><span class="linenos" data-linenos="7 "></span></a><span class="m">7</span>
</code></pre></div>
<h2 id="66-phoenix">6.6 验证 Phoenix 部署<a class="headerlink" href="#66-phoenix" title="Permanent link">&para;</a></h2>
<ul>
<li>前置条件：将 <code>phoenix-server-hbase-2.1-5.1.2.jar</code> 软链到所有机器的 CDH hbase lib 目录下并重启完成：</li>
</ul>
<div class="highlight"><pre><span></span><code><a id="__codelineno-21-1" name="__codelineno-21-1"></a><a href="#__codelineno-21-1"><span class="linenos" data-linenos=" 1 "></span></a>/opt/cloudera/parcels/phoenix/bin/sqlline.py
<a id="__codelineno-21-2" name="__codelineno-21-2"></a><a href="#__codelineno-21-2"><span class="linenos" data-linenos=" 2 "></span></a>
<a id="__codelineno-21-3" name="__codelineno-21-3"></a><a href="#__codelineno-21-3"><span class="linenos" data-linenos=" 3 "></span></a><span class="m">0</span>: jdbc:phoenix:&gt; !ta
<a id="__codelineno-21-4" name="__codelineno-21-4"></a><a href="#__codelineno-21-4"><span class="linenos" data-linenos=" 4 "></span></a>+-----------+-------------+------------+--------------+---------+-----------+---------------------------+----------------+-------------+----------------+--------+
<a id="__codelineno-21-5" name="__codelineno-21-5"></a><a href="#__codelineno-21-5"><span class="linenos" data-linenos=" 5 "></span></a><span class="p">|</span> TABLE_CAT <span class="p">|</span> TABLE_SCHEM <span class="p">|</span> TABLE_NAME <span class="p">|</span>  TABLE_TYPE  <span class="p">|</span> REMARKS <span class="p">|</span> TYPE_NAME <span class="p">|</span> SELF_REFERENCING_COL_NAME <span class="p">|</span> REF_GENERATION <span class="p">|</span> INDEX_STATE <span class="p">|</span> IMMUTABLE_ROWS <span class="p">|</span> SALT_B <span class="p">|</span>
<a id="__codelineno-21-6" name="__codelineno-21-6"></a><a href="#__codelineno-21-6"><span class="linenos" data-linenos=" 6 "></span></a>+-----------+-------------+------------+--------------+---------+-----------+---------------------------+----------------+-------------+----------------+--------+
<a id="__codelineno-21-7" name="__codelineno-21-7"></a><a href="#__codelineno-21-7"><span class="linenos" data-linenos=" 7 "></span></a><span class="p">|</span>           <span class="p">|</span> SYSTEM      <span class="p">|</span> CATALOG    <span class="p">|</span> SYSTEM TABLE <span class="p">|</span>         <span class="p">|</span>           <span class="p">|</span>                           <span class="p">|</span>                <span class="p">|</span>             <span class="p">|</span> <span class="nb">false</span>          <span class="p">|</span> null   <span class="p">|</span>
<a id="__codelineno-21-8" name="__codelineno-21-8"></a><a href="#__codelineno-21-8"><span class="linenos" data-linenos=" 8 "></span></a><span class="p">|</span>           <span class="p">|</span> SYSTEM      <span class="p">|</span> CHILD_LINK <span class="p">|</span> SYSTEM TABLE <span class="p">|</span>         <span class="p">|</span>           <span class="p">|</span>                           <span class="p">|</span>                <span class="p">|</span>             <span class="p">|</span> <span class="nb">false</span>          <span class="p">|</span> null   <span class="p">|</span>
<a id="__codelineno-21-9" name="__codelineno-21-9"></a><a href="#__codelineno-21-9"><span class="linenos" data-linenos=" 9 "></span></a><span class="p">|</span>           <span class="p">|</span> SYSTEM      <span class="p">|</span> FUNCTION   <span class="p">|</span> SYSTEM TABLE <span class="p">|</span>         <span class="p">|</span>           <span class="p">|</span>                           <span class="p">|</span>                <span class="p">|</span>             <span class="p">|</span> <span class="nb">false</span>          <span class="p">|</span> null   <span class="p">|</span>
<a id="__codelineno-21-10" name="__codelineno-21-10"></a><a href="#__codelineno-21-10"><span class="linenos" data-linenos="10 "></span></a><span class="p">|</span>           <span class="p">|</span> SYSTEM      <span class="p">|</span> LOG        <span class="p">|</span> SYSTEM TABLE <span class="p">|</span>         <span class="p">|</span>           <span class="p">|</span>                           <span class="p">|</span>                <span class="p">|</span>             <span class="p">|</span> <span class="nb">true</span>           <span class="p">|</span> <span class="m">32</span>     <span class="p">|</span>
<a id="__codelineno-21-11" name="__codelineno-21-11"></a><a href="#__codelineno-21-11"><span class="linenos" data-linenos="11 "></span></a><span class="p">|</span>           <span class="p">|</span> SYSTEM      <span class="p">|</span> MUTEX      <span class="p">|</span> SYSTEM TABLE <span class="p">|</span>         <span class="p">|</span>           <span class="p">|</span>                           <span class="p">|</span>                <span class="p">|</span>             <span class="p">|</span> <span class="nb">true</span>           <span class="p">|</span> null   <span class="p">|</span>
<a id="__codelineno-21-12" name="__codelineno-21-12"></a><a href="#__codelineno-21-12"><span class="linenos" data-linenos="12 "></span></a><span class="p">|</span>           <span class="p">|</span> SYSTEM      <span class="p">|</span> SEQUENCE   <span class="p">|</span> SYSTEM TABLE <span class="p">|</span>         <span class="p">|</span>           <span class="p">|</span>                           <span class="p">|</span>                <span class="p">|</span>             <span class="p">|</span> <span class="nb">false</span>          <span class="p">|</span> null   <span class="p">|</span>
<a id="__codelineno-21-13" name="__codelineno-21-13"></a><a href="#__codelineno-21-13"><span class="linenos" data-linenos="13 "></span></a><span class="p">|</span>           <span class="p">|</span> SYSTEM      <span class="p">|</span> STATS      <span class="p">|</span> SYSTEM TABLE <span class="p">|</span>         <span class="p">|</span>           <span class="p">|</span>                           <span class="p">|</span>                <span class="p">|</span>             <span class="p">|</span> <span class="nb">false</span>          <span class="p">|</span> null   <span class="p">|</span>
<a id="__codelineno-21-14" name="__codelineno-21-14"></a><a href="#__codelineno-21-14"><span class="linenos" data-linenos="14 "></span></a><span class="p">|</span>           <span class="p">|</span> SYSTEM      <span class="p">|</span> TASK       <span class="p">|</span> SYSTEM TABLE <span class="p">|</span>         <span class="p">|</span>           <span class="p">|</span>                           <span class="p">|</span>                <span class="p">|</span>             <span class="p">|</span> <span class="nb">false</span>          <span class="p">|</span> null   <span class="p">|</span>
<a id="__codelineno-21-15" name="__codelineno-21-15"></a><a href="#__codelineno-21-15"><span class="linenos" data-linenos="15 "></span></a>+-----------+-------------+------------+--------------+---------+-----------+---------------------------+----------------+-------------+----------------+--------+
<a id="__codelineno-21-16" name="__codelineno-21-16"></a><a href="#__codelineno-21-16"><span class="linenos" data-linenos="16 "></span></a><span class="m">0</span>: jdbc:phoenix:&gt;
<a id="__codelineno-21-17" name="__codelineno-21-17"></a><a href="#__codelineno-21-17"><span class="linenos" data-linenos="17 "></span></a><span class="m">0</span>: jdbc:phoenix:&gt; CREATE TABLE <span class="s2">&quot;test&quot;</span>.<span class="s2">&quot;t_user&quot;</span> <span class="o">(</span>
<a id="__codelineno-21-18" name="__codelineno-21-18"></a><a href="#__codelineno-21-18"><span class="linenos" data-linenos="18 "></span></a>    <span class="s2">&quot;ROW&quot;</span> VARCHAR PRIMARY KEY,
<a id="__codelineno-21-19" name="__codelineno-21-19"></a><a href="#__codelineno-21-19"><span class="linenos" data-linenos="19 "></span></a>    <span class="s2">&quot;f1&quot;</span>.<span class="s2">&quot;firstName&quot;</span> VARCHAR <span class="o">(</span><span class="m">16</span><span class="o">)</span>,
<a id="__codelineno-21-20" name="__codelineno-21-20"></a><a href="#__codelineno-21-20"><span class="linenos" data-linenos="20 "></span></a>    <span class="s2">&quot;f1&quot;</span>.<span class="s2">&quot;lastName&quot;</span> VARCHAR,
<a id="__codelineno-21-21" name="__codelineno-21-21"></a><a href="#__codelineno-21-21"><span class="linenos" data-linenos="21 "></span></a>    <span class="s2">&quot;f1&quot;</span>.<span class="s2">&quot;age&quot;</span> INTEGER,
<a id="__codelineno-21-22" name="__codelineno-21-22"></a><a href="#__codelineno-21-22"><span class="linenos" data-linenos="22 "></span></a>    <span class="s2">&quot;f2&quot;</span>.<span class="s2">&quot;money&quot;</span> DECIMAL<span class="o">(</span><span class="m">10</span>,2<span class="o">)</span>,
<a id="__codelineno-21-23" name="__codelineno-21-23"></a><a href="#__codelineno-21-23"><span class="linenos" data-linenos="23 "></span></a>    <span class="s2">&quot;f2&quot;</span>.<span class="s2">&quot;idcard&quot;</span> BIGINT
<a id="__codelineno-21-24" name="__codelineno-21-24"></a><a href="#__codelineno-21-24"><span class="linenos" data-linenos="24 "></span></a><span class="o">)</span> <span class="nv">COLUMN_ENCODED_BYTES</span> <span class="o">=</span> <span class="m">0</span><span class="p">;</span>
<a id="__codelineno-21-25" name="__codelineno-21-25"></a><a href="#__codelineno-21-25"><span class="linenos" data-linenos="25 "></span></a>
<a id="__codelineno-21-26" name="__codelineno-21-26"></a><a href="#__codelineno-21-26"><span class="linenos" data-linenos="26 "></span></a><span class="m">0</span>: jdbc:phoenix:&gt; UPSERT INTO <span class="s2">&quot;test&quot;</span>.<span class="s2">&quot;t_user&quot;</span> <span class="o">(</span><span class="s2">&quot;ROW&quot;</span>, <span class="s2">&quot;firstName&quot;</span>, <span class="s2">&quot;lastName&quot;</span>, <span class="s2">&quot;age&quot;</span>, <span class="s2">&quot;money&quot;</span>, <span class="s2">&quot;idcard&quot;</span><span class="o">)</span> VALUES <span class="o">(</span><span class="s1">&#39;12301062022,admin,1&#39;</span>, <span class="s1">&#39;foo&#39;</span>, <span class="s1">&#39;bar&#39;</span>, <span class="m">18</span>, <span class="m">9999999</span>.01, <span class="m">42310197001011212</span><span class="o">)</span><span class="p">;</span>
<a id="__codelineno-21-27" name="__codelineno-21-27"></a><a href="#__codelineno-21-27"><span class="linenos" data-linenos="27 "></span></a><span class="m">0</span>: jdbc:phoenix:&gt;
<a id="__codelineno-21-28" name="__codelineno-21-28"></a><a href="#__codelineno-21-28"><span class="linenos" data-linenos="28 "></span></a><span class="m">0</span>: jdbc:phoenix:&gt; <span class="k">select</span> * from <span class="s2">&quot;test&quot;</span>.<span class="s2">&quot;t_user&quot;</span><span class="p">;</span>
</code></pre></div>
<h2 id="7-faq">7. FAQ<a class="headerlink" href="#7-faq" title="Permanent link">&para;</a></h2>
<h4 id="71-install-cdh-parcels-block-agents-from-heartbeating-to-a-cloudera-manager-with-different-uuid-until-agent-restart">7.1 Install CDH Parcels 包时报错，提示主机运行状况不良，错误原因官方文档解释：<a href="https://docs.cloudera.com/documentation/enterprise/release-notes/topics/cm_rn_known_issues.html#rn_OPSAPS-34847">Block agents from heartbeating to a Cloudera Manager with different UUID until agent restart</a><a class="headerlink" href="#71-install-cdh-parcels-block-agents-from-heartbeating-to-a-cloudera-manager-with-different-uuid-until-agent-restart" title="Permanent link">&para;</a></h4>
<ul>
<li>解决</li>
</ul>
<div class="highlight"><pre><span></span><code><a id="__codelineno-22-1" name="__codelineno-22-1"></a><a href="#__codelineno-22-1"><span class="linenos" data-linenos="1 "></span></a><span class="c1"># 删除所有有问题服务器的agent目录下面的cm_guid文件，并重启失败节点的agent服务恢复</span>
<a id="__codelineno-22-2" name="__codelineno-22-2"></a><a href="#__codelineno-22-2"><span class="linenos" data-linenos="2 "></span></a><span class="nb">cd</span> /var/lib/cloudera-scm-agent
<a id="__codelineno-22-3" name="__codelineno-22-3"></a><a href="#__codelineno-22-3"><span class="linenos" data-linenos="3 "></span></a><span class="c1"># 删除cm-guid文件</span>
<a id="__codelineno-22-4" name="__codelineno-22-4"></a><a href="#__codelineno-22-4"><span class="linenos" data-linenos="4 "></span></a>rm -f cm_guid
<a id="__codelineno-22-5" name="__codelineno-22-5"></a><a href="#__codelineno-22-5"><span class="linenos" data-linenos="5 "></span></a><span class="c1"># 重启agent服务，重启后会自动安装失败的agent</span>
<a id="__codelineno-22-6" name="__codelineno-22-6"></a><a href="#__codelineno-22-6"><span class="linenos" data-linenos="6 "></span></a>service cloudera-scm-agent restart
</code></pre></div>
<h4 id="72-cannot-find-cdhs-bigtop-detect-javahome">7.2 启动组件时报错 <code>Cannot find CDH's bigtop-detect-javahome.</code><a class="headerlink" href="#72-cannot-find-cdhs-bigtop-detect-javahome" title="Permanent link">&para;</a></h4>
<ul>
<li>解决：<a href="https://www.bbsmax.com/A/kvJ36j8Dzg/">bbsmax.com/A/kvJ36j8Dzg/</a></li>
</ul>
<div class="highlight"><pre><span></span><code><a id="__codelineno-23-1" name="__codelineno-23-1"></a><a href="#__codelineno-23-1"><span class="linenos" data-linenos="1 "></span></a>sudo mkdir -p /usr/libexec/bigtop-utils/
<a id="__codelineno-23-2" name="__codelineno-23-2"></a><a href="#__codelineno-23-2"><span class="linenos" data-linenos="2 "></span></a><span class="c1"># 如：</span>
<a id="__codelineno-23-3" name="__codelineno-23-3"></a><a href="#__codelineno-23-3"><span class="linenos" data-linenos="3 "></span></a>sudo <span class="nb">echo</span> <span class="s1">&#39;export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.322.b06-1.el7_9.x86_64&#39;</span> &gt; /usr/libexec/bigtop-utils/bigtop-detect-javahome
<a id="__codelineno-23-4" name="__codelineno-23-4"></a><a href="#__codelineno-23-4"><span class="linenos" data-linenos="4 "></span></a>
<a id="__codelineno-23-5" name="__codelineno-23-5"></a><a href="#__codelineno-23-5"><span class="linenos" data-linenos="5 "></span></a><span class="c1"># 推荐设置这里，所有组件生效</span>
<a id="__codelineno-23-6" name="__codelineno-23-6"></a><a href="#__codelineno-23-6"><span class="linenos" data-linenos="6 "></span></a>sudo <span class="nb">echo</span> <span class="s1">&#39;export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.322.b06-1.el7_9.x86_64&#39;</span> &gt; /etc/default/bigtop-utils
<a id="__codelineno-23-7" name="__codelineno-23-7"></a><a href="#__codelineno-23-7"><span class="linenos" data-linenos="7 "></span></a><span class="c1"># 参见源码：vim +21 /opt/cloudera/parcels/CDH-6.3.1-1.cdh6.3.1.p0.1470567/bin/bigtop-detect-javahome</span>
</code></pre></div>
<h4 id="73-flink">7.3 如何集成第三方组件 Flink？<a class="headerlink" href="#73-flink" title="Permanent link">&para;</a></h4>
<ul>
<li>
<p>7.3.1 首先，Cloudera Manager 是采用 parcel 包进行离线统一部署的，且 CDH 默认是不支持 Flink/Phoneix 集成的(CDP支持)，因此必须自己构建 flink 的 parcel 包，请参见我这个项目 <a href="https://gitee.com/wl4g-collect/flink-parcel-generator">(使用 cm_ext 构建 CDH 6.3.1 + Flink 1.14.4 parcel 包) gitee.com/wl4g-collect/flink-parcel-generator</a></p>
</li>
<li>
<p>7.3.2 在准备加载 flink parcel 之前，请先将 parcel 包覆盖上传至 <code>/opt/cloudera/parcel-repo/</code>，如：
<div class="highlight"><pre><span></span><code><a id="__codelineno-24-1" name="__codelineno-24-1"></a><a href="#__codelineno-24-1"><span class="linenos" data-linenos="1 "></span></a>tree /opt/cloudera/parcel-repo/
<a id="__codelineno-24-2" name="__codelineno-24-2"></a><a href="#__codelineno-24-2"><span class="linenos" data-linenos="2 "></span></a>├── CDH-6.3.1-1.cdh6.3.1.p0.1470567-el7.parcel
<a id="__codelineno-24-3" name="__codelineno-24-3"></a><a href="#__codelineno-24-3"><span class="linenos" data-linenos="3 "></span></a>├── CDH-6.3.1-1.cdh6.3.1.p0.1470567-el7.parcel.sha
<a id="__codelineno-24-4" name="__codelineno-24-4"></a><a href="#__codelineno-24-4"><span class="linenos" data-linenos="4 "></span></a>├── CDH-6.3.1-1.cdh6.3.1.p0.1470567-el7.parcel.sha1
<a id="__codelineno-24-5" name="__codelineno-24-5"></a><a href="#__codelineno-24-5"><span class="linenos" data-linenos="5 "></span></a>├── CDH-6.3.1-1.cdh6.3.1.p0.1470567-el7.parcel.torrent
<a id="__codelineno-24-6" name="__codelineno-24-6"></a><a href="#__codelineno-24-6"><span class="linenos" data-linenos="6 "></span></a>├── flink-1.14.4-bin-scala_2.11-el7.parcel
<a id="__codelineno-24-7" name="__codelineno-24-7"></a><a href="#__codelineno-24-7"><span class="linenos" data-linenos="7 "></span></a>├── flink-1.14.4-bin-scala_2.11-el7.parcel.sha
<a id="__codelineno-24-8" name="__codelineno-24-8"></a><a href="#__codelineno-24-8"><span class="linenos" data-linenos="8 "></span></a>├── flink-1.14.4-bin-scala_2.11-el7.parcel.torrent
<a id="__codelineno-24-9" name="__codelineno-24-9"></a><a href="#__codelineno-24-9"><span class="linenos" data-linenos="9 "></span></a>└── manifest.json
</code></pre></div>
然后在 “主机” -&gt; “Parcel” -&gt; “检查新 Parcel” 执行加载，同时必须将 csd 包手动上传到 <code>/opt/cloudera/csd/flink_on_yarn-1.14.4.jar</code> 和 <code>/opt/cloudera/csd/flink_standalone-1.14.4.jar</code>，否则可能就算分片且激活成功 flink parcel 包，但是给集群添加服务时还可能选不了 flink。</p>
</li>
<li>
<p>7.3.3 集成完 Flink 后安装目录长这样：</p>
</li>
</ul>
<p><details style="cursor:pointer;;padding-left:2%">
<summary style="color:#007acc;">[展开] /opt/cloudera 目录树</summary>
<code><pre>
/opt/cloudera/
├── cm
│  ├── actionItem_en.properties
...
├── cm-agent
│  ├── bin
│  ├── cm_version.properties
│  ├── lib64 -> lib
...
├── csd
│  ├── flink_on_yarn-1.14.4.jar
│  └── flink_standalone-1.14.4.jar
...
├── parcel-cache
│  ├── CDH-6.3.1-1.cdh6.3.1.p0.1470567-el7.parcel.torrent
│  └── flink-1.14.4-bin-scala_2.11-el7.parcel.torrent
...
├── parcel-repo
│  ├── CDH-6.3.1-1.cdh6.3.1.p0.1470567-el7.parcel
│  ├── CDH-6.3.1-1.cdh6.3.1.p0.1470567-el7.parcel.sha
│  ├── CDH-6.3.1-1.cdh6.3.1.p0.1470567-el7.parcel.torrent
│  ├── flink-1.14.4-bin-scala_2.11-el7.parcel  ## 扩展的 Flink 包
│  ├── flink-1.14.4-bin-scala_2.11-el7.parcel.sha
│  ├── flink-1.14.4-bin-scala_2.11-el7.parcel.torrent
│  └── manifest.json
...
└── parcels
    │── CDH -> CDH-6.3.1-1.cdh6.3.1.p0.1470567
    │── CDH-6.3.1-1.cdh6.3.1.p0.1470567
    │    ├── bin
    │    │  ├── bigtop-detect-javahome -> ../lib/bigtop-utils/bigtop-detect-javahome
    │    │  ├── flume-ng
    │    │  ├── hadoop
    │    │  ├── hbase
    │    │  ├── hdfs
    │    │  ├── hive
    │    │  ├── kafka-acls
    │    │  ├── mapred
    │    │  ├── oozie
    │    │  ├── pig
    │    │  ├── spark-submit
    │    │  ├── sqoop
    │    │  ├── yarn
    │    │  ├── zookeeper-server
    │    ...
    │    ├── etc
    │    │  ├── hadoop
    │    │  ├── hadoop-httpfs
    │    │  ├── hbase
    │    │  ├── hbase-solr
    │    │  ├── hive
    │    │  ├── kafka
    │    │  ├── solr
    │    │  ├── spark
    │    │  ├── sqoop
    │    │  └── zookeeper
    │   ...
    │    ├── jars
    │    │  ├── avro-mapred-1.8.2-cdh6.3.1-hadoop2.jar
    │    │  ├── hadoop-aliyun-3.0.0-cdh6.3.1.jar
    │    │  ├── hbase-zookeeper-2.1.0-cdh6.3.1.jar
    │    │  ├── hive-storage-api-2.1.1-cdh6.3.1.jar
    │    │  ├── lucene-analyzers-common-7.4.0-cdh6.3.1.jar
    │    │  ├── netty-resolver-4.1.17.Final.jar
    │    │  ├── oozie-tools-5.1.0-cdh6.3.1.jar
    │    │  ├── parquet-tools-1.9.0-cdh6.3.1.jar
    │    │  ├── spark-yarn_2.11-2.4.0-cdh6.3.1.jar
    │    │            ├── zookeeper-3.4.6.jar
    │    ...
    │    ├── lib
    │    │  ├── avro
    │    │  ├── bigtop-utils
    │    │  ├── flume-ng
    │    │  ├── hadoop
    │    │  ├── hadoop-hdfs
    │    │  ├── hadoop-mapreduce
    │    │  ├── hadoop-yarn
    │    │  ├── hbase
    │    │  ├── hive
    │    │  ├── hue
    │    │  ├── impala
    │    │  ├── kafka
    │    │  ├── pig
    │    │  ├── solr
    │    │  ├── spark
    │    │  ├── sqoop
    │    │  └── zookeeper-native
    │    ...
    │    ├── lib64
    │    │  ├── libhdfs.so -> libhdfs.so.0.0.0
    │    │  ├── libhdfs.so.0.0.0
    │    │  ├── libzookeeper_st.so -> libzookeeper_st.so.2.0.0
    │    ...
    │    ├── libexec
    │    │  └── bigtop-utils
    │    ├── meta
    │    │  ├── alternatives.json
    │    │  ├── parcel.json
    │    └── share
    │        ├── doc
    │    ...
    ├── flink -> flink-1.14.4-bin-scala_2.11    ## 扩展的 Flink 包解压后安装目录
    └── flink-1.14.4-bin-scala_2.11
        ├── lib
        │  └── flink
        │  ├── bin
        │  ├── conf
        │  ├── examples
        │  ├── lib
        │  ├── LICENSE
        │  ├── licenses
        │  ├── log
        │  ├── NOTICE
        │  ├── opt
        │  ├── plugins
        │  └── README.txt
        └── meta
            ├── flink_env.sh
            ├── parcel.json
            └── permissions.json
</pre>
</code>
</details>
</p>

<ul>
<li>
<p>7.3.4 <strong><em>如果以上包都正确上传和配置后，依然在给集群添加服务时选择不了 flink，那么就重启 <code>systemctl restart cloudera-scm-server</code>，因为有RAM缓存，通常这都会OK。</em></strong></p>
</li>
<li>
<p>相关资料1：<a href="https://chowdera.com/2020/12/20201213073419954w.html">chowdera.com/2020/12/20201213073419954w.html</a></p>
</li>
</ul>
<h4 id="74-flink-yarn-noclassdeffounderror-orgapachehadoopyarnexceptionsyarnexception">7.4 当启动集成 Flink yarn 组件报错 <code>NoClassDefFoundError: org/apache/hadoop/yarn/exceptions/YarnException</code><a class="headerlink" href="#74-flink-yarn-noclassdeffounderror-orgapachehadoopyarnexceptionsyarnexception" title="Permanent link">&para;</a></h4>
<ul>
<li>
<p>解决方案1：需手动将 flink 依赖的 hadoop 版本上传到 <code>flink/lib</code> 目录即可，<strong>注：</strong> 但亲测发现用 hadoop-3-uber-3.1.1 的包又会报错 <code>NoClassDefFoundError: org/apache/commons/cli/CommandLine</code>，最终尝试用如下 hadoop-2-uber-2.8.3 竟然OK。</p>
</li>
<li>
<p><a href="https://mvnrepository.com/artifact/org.apache.flink/flink-shaded-hadoop-3-uber/3.1.1.7.2.9.0-173-9.0">flink-shaded-hadoop-3-uber-3.1.1.7.2.9.0-173-9.0.jar</a></p>
</li>
<li>
<p><a href="https://repo1.maven.org/maven2/org/apache/flink/flink-shaded-hadoop-2-uber/2.8.3-10.0/flink-shaded-hadoop-2-uber-2.8.3-10.0.jar">flink-shaded-hadoop-2-uber-2.8.3-10.0.jar</a></p>
</li>
<li>
<p>解决方案2：如果flink版本&gt;=1.12.0，可以在：“Flink yarn” -&gt; 配置 -&gt; 高级 -&gt; ”Flink yarn 服务环境高级配置代码段（安全阀） Flink yarn（服务范围）” 可以添加如下配置（重启 Flink yarn 服务就不报错）：</p>
</li>
</ul>
<div class="highlight"><pre><span></span><code><a id="__codelineno-25-1" name="__codelineno-25-1"></a><a href="#__codelineno-25-1"><span class="linenos" data-linenos="1 "></span></a><span class="nv">HADOOP_USER_NAME</span><span class="o">=</span>flink
<a id="__codelineno-25-2" name="__codelineno-25-2"></a><a href="#__codelineno-25-2"><span class="linenos" data-linenos="2 "></span></a><span class="nv">HADOOP_CONF_DIR</span><span class="o">=</span>/etc/hadoop/conf
<a id="__codelineno-25-3" name="__codelineno-25-3"></a><a href="#__codelineno-25-3"><span class="linenos" data-linenos="3 "></span></a><span class="nv">HADOOP_HOME</span><span class="o">=</span>/opt/cloudera/parcels/CDH
<a id="__codelineno-25-4" name="__codelineno-25-4"></a><a href="#__codelineno-25-4"><span class="linenos" data-linenos="4 "></span></a><span class="nv">HADOOP_CLASSPATH</span><span class="o">=</span>/opt/cloudera/parcels/CDH/jars/*
</code></pre></div>
<h3 id="75-safemode-access-denied-for-user-root-superuser-privilege-is-required">7.5 <code>safemode: Access denied for user root. Superuser privilege is required</code> 何解？<a class="headerlink" href="#75-safemode-access-denied-for-user-root-superuser-privilege-is-required" title="Permanent link">&para;</a></h3>
<p>实际案例：启动 HBase/Spark/Flink 等时发现报错 <code>Name node is in safe mode</code>，然后执行 <code>hdfs dfsadmin -safemode leave</code> 又发现报错 <code>safemode: Access denied for user root. Superuser privilege is required</code></p>
<p>解决:<code>sudo -u hdfs hdfs dfsadmin -safemode leave</code>，参考：<a href="https://community.cloudera.com/t5/Support-Questions/safemode-Access-denied-for-user-cloudera-Superuser-privilege/td-p/2207">safemode-Access-denied-for-user-cloudera-Superuser-privilege</a></p>
<h3 id="76-scala_212-optclouderaparcelsflink-1144-bin-scala_212libflinkbinstart-scala-shellsh-error-could-not-find-or-load-main-class-orgapacheflinkapiscalaflinkshell">7.6 如果是用的 scala_2.12 对应版本，则启动 <code>/opt/cloudera/parcels/flink-1.14.4-bin-scala_2.12/lib/flink/bin/start-scala-shell.sh</code> 时报错：<code>Error: Could not find or load main class org.apache.flink.api.scala.FlinkShell</code><a class="headerlink" href="#76-scala_212-optclouderaparcelsflink-1144-bin-scala_212libflinkbinstart-scala-shellsh-error-could-not-find-or-load-main-class-orgapacheflinkapiscalaflinkshell" title="Permanent link">&para;</a></h3>
<ul>
<li>
<p>原因：是因为基于 Scala 2.12 版本构建的 Flink 尚不支持 Flink Scala shell，但想使用 Flink Scala shell，应该使用基于 Scala 2.11 构建的 Flink 二进制文件，到 flink-1.14.4 为止还不支持，参见：<a href="https://github.com/apache/flink/blob/release-1.14.4/flink-dist/pom.xml#L434">github.com/apache/flink/blob/release-1.14.4/flink-dist/pom.xml#L434</a></p>
</li>
<li>
<p><a href="https://blog.csdn.net/u012551524/article/details/113662293">强制自行编译 scala_2.12 的方案（未验证）</a></p>
</li>
</ul>
<h3 id="77-hadoop-hbase-trace">7.7 Hadoop / HBase Trace 配置使用?<a class="headerlink" href="#77-hadoop-hbase-trace" title="Permanent link">&para;</a></h3>
<ul>
<li>参见1：<a href="https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-common/Tracing.html">hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-common/Tracing.html</a></li>
<li>参见2：<a href="https://hbase.apache.org/book.html#_enable_tracing">hbase.apache.org/book.html#_enable_tracing</a> Replace <a href="https://htrace.apache.org">htrace</a> and upgrade to <a href="https://opentelemetry.io">opentelemetry</a></li>
<li>参见3：<a href="https://archive.cloudera.com/cloudera-labs/htrace/redhat/7/x86_64/htrace/">cloudera labs for htrace repo</a></li>
</ul>
<h3 id="78-rpm-ubuntu">7.8 在只有 rpm 包的情况下如何在 ubuntu 下部署？<a class="headerlink" href="#78-rpm-ubuntu" title="Permanent link">&para;</a></h3>
<ul>
<li>使用 <code>alien</code> 将 rpm 转为 deb<blockquote>
<p>注：目前在测试环境能部署成功，但发现会有一些小问题，如执行 <code>dpki -i cloudera-*.deb</code> 时报错无法创建用户 <code>clouera-scm</code>，需提前手动创建即可。</p>
</blockquote>
</li>
</ul>
<div class="highlight"><pre><span></span><code><a id="__codelineno-26-1" name="__codelineno-26-1"></a><a href="#__codelineno-26-1"><span class="linenos" data-linenos="1 "></span></a>sudo apt install -y alien
<a id="__codelineno-26-2" name="__codelineno-26-2"></a><a href="#__codelineno-26-2"><span class="linenos" data-linenos="2 "></span></a>sudo alien cloudera-*.rpm -v <span class="c1"># 一键转换</span>
</code></pre></div>
<h3 id="79-flink-yarn-optclouderaparcelsflinklibflinkbinflink-yarnsh-17-rotatelogfileswithprefix-command-not-found">7.9 当启动 Flink-yarn 时报错 <code>/opt/cloudera/parcels/flink/lib/flink/bin/flink-yarn.sh +17  rotateLogFilesWithPrefix command not found</code><a class="headerlink" href="#79-flink-yarn-optclouderaparcelsflinklibflinkbinflink-yarnsh-17-rotatelogfileswithprefix-command-not-found" title="Permanent link">&para;</a></h3>
<ul>
<li>通常这是由于 flink-yarn 默认配置了两项：<code>security.kerberos.login.keytab</code> / <code>security.kerberos.login.principal</code>，如果不想启用 kerberos 认证，则直接设为空即可，同时还应设置 HADOOP 相关环节变量，其选项名为：<code>“Flink-yarn 服务环境高级配置代码段（安全阀）
Flink-yarn（服务范围）”</code>:
<div class="highlight"><pre><span></span><code><a id="__codelineno-27-1" name="__codelineno-27-1"></a><a href="#__codelineno-27-1"><span class="linenos" data-linenos="1 "></span></a><span class="nv">HADOOP_USER_NAME</span><span class="o">=</span>flink
<a id="__codelineno-27-2" name="__codelineno-27-2"></a><a href="#__codelineno-27-2"><span class="linenos" data-linenos="2 "></span></a><span class="nv">HADOOP_CONF_DIR</span><span class="o">=</span>/etc/hadoop/conf
<a id="__codelineno-27-3" name="__codelineno-27-3"></a><a href="#__codelineno-27-3"><span class="linenos" data-linenos="3 "></span></a><span class="nv">HADOOP_HOME</span><span class="o">=</span>/opt/cloudera/parcels/CDH
<a id="__codelineno-27-4" name="__codelineno-27-4"></a><a href="#__codelineno-27-4"><span class="linenos" data-linenos="4 "></span></a><span class="nv">HADOOP_CLASSPATH</span><span class="o">=</span>/opt/cloudera/parcels/CDH/jars/*
</code></pre></div></li>
</ul>
<h3 id="710-cdh-cloudera-manager-agent">7.10 创建 CDH 集群安装时部分节点总是提示：复制失败或无法安装 cloudera-manager-agent？<a class="headerlink" href="#710-cdh-cloudera-manager-agent" title="Permanent link">&para;</a></h3>
<ul>
<li>请检测改节点的 repo 源配置是否正确：<code>ls -al /etc/yum.repo.d/</code></li>
</ul>
<h3 id="711-cdh-agent">7.11 创建 CDH 集群安装时部分节点总是提示：安装失败。 无法接收 Agent 发出的检测信号？<a class="headerlink" href="#711-cdh-agent" title="Permanent link">&para;</a></h3>
<ul>
<li>有这么几种可能：
1、Python文件不匹配；
2、日志文件不存在，在config.ini中把log_file放开；
3、/etc/hosts/中主机和ip配置问题；
4、防火墙是否关闭，ubuntu是ufw disable；
5、端口配置，config.ini中端口是否配置的为7182；
6、集群时间是否同步，安装ntp同步时间；
7、ssh私钥的问题-----我现在正在查这个问题呢，前边都配完了，但是仍然无法检测到信号，我没有使用私钥，不知道是不是跟这个有关系；
8，如果此目的无权限则应该授权：<code>sudo chmod -R cloudera-scm:cloudera-scm /var/log/cloudera-scm-agent/</code></li>
</ul>
<h3 id="712-cloudera-scm-server">7.12 如何修改 cloudera-scm-server 日志级别？<a class="headerlink" href="#712-cloudera-scm-server" title="Permanent link">&para;</a></h3>
<ul>
<li>
<p>参见1：<a href="https://xiyoulaoyuanjia.cn/?p=476">cloudera manager debug 日志级别</a></p>
</li>
<li>
<p>参见2：https://community.cloudera.com/t5/Support-Questions/How-to-enable-debug-logging-for-Cloudera-Manager-server/m-p/79591</p>
</li>
<li>
<p>方法1：在页面上直接改：http://your_ip:7180/cmf/debug/logLevel</p>
</li>
</ul>
<h3 id="713-cmf">7.13 如果您是面向私有云部署的企业运开人员，可能需要经常新部署集群，能否快速导入已部署集群 cmf 库？<a class="headerlink" href="#713-cmf" title="Permanent link">&para;</a></h3>
<ul>
<li>应该是不行的，一是，不同集群的 IP 不同，即使导入了也得手动改，而且 cmf 库的表关联关系不开源，极易出错几乎不可能；二是，决定性不可行的，cmf 库中只是配置后的状态数据，实际上会对应每个节点真实安装的jar包和脚本，否则光导入一堆数据是没用的，因此如果需大量部署私有云集群，目前是暂未有啥可靠的方法，主要原因就是不开源，否则知道表关联关系是完全可导入然后手动修改如 IP 部分，基于这个需求开源的 Ambari 更有优势。</li>
</ul>
<h3 id="714-cdh-kafka-zookeeperchroot-kafka">7.14 当修改了 CDH kafka 配置项 <code>zookeeper.chroot</code> 默认值时，如改为 <code>/kafka</code>，会导致查不到主题？<a class="headerlink" href="#714-cdh-kafka-zookeeperchroot-kafka" title="Permanent link">&para;</a></h3>
<h4 id="cdh631-kafka_211-221">环境：CDH6.3.1 / kafka_2.11-2.2.1<a class="headerlink" href="#cdh631-kafka_211-221" title="Permanent link">&para;</a></h4>
<h4 id="_1">现象：<a class="headerlink" href="#_1" title="Permanent link">&para;</a></h4>
<ul>
<li>
<p>操作1：笔者为公司某客户私有云CDH集群，由于资源问题部分组件选择了混合，因此修改了 kafka 在 zookeeper 的根路径配置项 <code>zookeeper.chroot</code> 为 <code>/kafka</code>，但在实测集群可用性时发现，执行创建 topc <code>kafka-topics --zookeeper 127.0.0.1:2181 --create --topic safeclound_ammeter --partitions 100 --replication-factor 5</code> 成功，但执行查看主题 <code>kafka-topics --zookeeper 127.0.0.1:2181 --list</code> 查不到任何 topics。</p>
</li>
<li>
<p>操作2：尝试了在控制台启动生产者 <code>kafka-console-producer --broker-list 127.0.0.1:9092 --topic safeclound_ammeter --property parse.key=true --property key.separator=:</code> 和消费者 <code>kafka-console-consumer --bootstrap-server 127.0.0.1:9092 --topic safeclound_ammeter</code> 进行测试发现竟然能生产也能消费此 topic 的数据，查看了 CM 生成的配置是有的：
<div class="highlight"><pre><span></span><code><a id="__codelineno-28-1" name="__codelineno-28-1"></a><a href="#__codelineno-28-1"><span class="linenos" data-linenos="1 "></span></a>cat /etc/kafka/conf/kafka-client.conf
<a id="__codelineno-28-2" name="__codelineno-28-2"></a><a href="#__codelineno-28-2"><span class="linenos" data-linenos="2 "></span></a>zookeeper.connect<span class="o">=</span>emr-master-1:2181,emr-master-2:2181,emr-worker-1:2181,emr-worker-2:2181,emr-worker-3:2181/kafka
</code></pre></div>
同时也查看了 zk 中生成路径也是正常在：<code>/kafka/xxx</code> 下。</p>
</li>
<li>
<p>操作3：又尝试将 <code>zookeeper.chroot</code> 改为空，重启，再执行 <code>kafka-topics --zookeeper 127.0.0.1:2181 --list</code> 是能查到的</p>
</li>
<li>
<p>由此分析应该是有 BUG，太忙就没必要花时间深入研究到底是 CDH 配置的键错误还是 kafka 内部错误，总之，通常都会使用默认空值就不会有任何问题，再就是默认开启了 producer 自动创建主题，那么只要是生产过数据的 topic 就都能查到了。当然在 kafka-manager 上也能查到，cmak 相关部署参见：https://blogs.wl4g.com/archives/1485。</p>
</li>
</ul>
<h3 id="715-flink-yarn-varlogflinkflink-yarnout-orgapacheflinkshadedzookeeper3orgapachezookeeperkeeperexceptionunimplementedexception-keepererrorcode-unimplemented-for-flinkcluster_yarn">7.15 启动 flink-yarn 服务时查看 <code>/var/log/flink/flink-yarn.out</code> 报错 <code>org.apache.flink.shaded.zookeeper3.org.apache.zookeeper.KeeperException$UnimplementedException: KeeperErrorCode = Unimplemented for /flink/cluster_yarn</code><a class="headerlink" href="#715-flink-yarn-varlogflinkflink-yarnout-orgapacheflinkshadedzookeeper3orgapachezookeeperkeeperexceptionunimplementedexception-keepererrorcode-unimplemented-for-flinkcluster_yarn" title="Permanent link">&para;</a></h3>
<ul>
<li>解决：手动创建 flink 在 zookeeper 的目录然后重启</li>
</ul>
<div class="highlight"><pre><span></span><code><a id="__codelineno-29-1" name="__codelineno-29-1"></a><a href="#__codelineno-29-1"><span class="linenos" data-linenos="1 "></span></a>zookeeper-client
<a id="__codelineno-29-2" name="__codelineno-29-2"></a><a href="#__codelineno-29-2"><span class="linenos" data-linenos="2 "></span></a>create /flink/cluster_yarn
</code></pre></div>
<h2 id="8">8. 参考资料<a class="headerlink" href="#8" title="Permanent link">&para;</a></h2>
<ul>
<li><a href="https://blog.csdn.net/u010520724/article/details/110635922">CDH 6.3 大数据平台搭建: blog.csdn.net/u010520724/article/details/110635922</a></li>
</ul>

              
            </article>
          </div>
        </div>
        
      </main>
      
        <footer class="md-footer">
  
  <div class="md-footer-meta md-typeset">
    <div class="md-footer-meta__inner md-grid">
      <div class="md-copyright">
  
  
    Made with
    <a href="https://squidfunk.github.io/mkdocs-material/" target="_blank" rel="noopener">
      Material for MkDocs
    </a>
  
</div>
      
    </div>
  </div>
</footer>
      
    </div>
    <div class="md-dialog" data-md-component="dialog">
      <div class="md-dialog__inner md-typeset"></div>
    </div>
    <script id="__config" type="application/json">{"base": "../../../..", "features": ["search.suggest", "search.highlight", "navigation.tabs", "navigation.expand", "toc.follow", "toc.integrate"], "search": "../../../../assets/javascripts/workers/search.5e67fbfe.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "version": {"default": "latest", "provider": "mike"}}</script>
    
    
      <script src="../../../../assets/javascripts/bundle.c44cc438.min.js"></script>
      
        <script src="../../../../static/js/util.js"></script>
      
    
  </body>
</html>